2024-12-08 03:58:23,348 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 03:58:23,362 main DEBUG Took 0.012030 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 03:58:23,363 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 03:58:23,363 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 03:58:23,364 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 03:58:23,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,373 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 03:58:23,385 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,386 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,387 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,388 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,390 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,391 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,392 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,393 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,394 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,394 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,395 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,395 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,396 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,396 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,397 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,397 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,398 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 03:58:23,399 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,399 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 03:58:23,401 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 03:58:23,402 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 03:58:23,405 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 03:58:23,405 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 03:58:23,406 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 03:58:23,407 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 03:58:23,416 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 03:58:23,419 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 03:58:23,421 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 03:58:23,421 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 03:58:23,421 main DEBUG createAppenders(={Console}) 2024-12-08 03:58:23,422 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-08 03:58:23,423 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 03:58:23,423 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-08 03:58:23,423 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 03:58:23,424 main DEBUG OutputStream closed 2024-12-08 03:58:23,424 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 03:58:23,424 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 03:58:23,424 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-08 03:58:23,484 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 03:58:23,486 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 03:58:23,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 03:58:23,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 03:58:23,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 03:58:23,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 03:58:23,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 03:58:23,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 03:58:23,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 03:58:23,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 03:58:23,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 03:58:23,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 03:58:23,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 03:58:23,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 03:58:23,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 03:58:23,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 03:58:23,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 03:58:23,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 03:58:23,495 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 03:58:23,495 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-08 03:58:23,495 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 03:58:23,496 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-08T03:58:23,690 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9 2024-12-08 03:58:23,692 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 03:58:23,692 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T03:58:23,700 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-08T03:58:23,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T03:58:23,729 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff, deleteOnExit=true 2024-12-08T03:58:23,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T03:58:23,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/test.cache.data in system properties and HBase conf 2024-12-08T03:58:23,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T03:58:23,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/hadoop.log.dir in system properties and HBase conf 2024-12-08T03:58:23,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T03:58:23,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T03:58:23,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T03:58:23,810 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T03:58:23,890 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T03:58:23,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T03:58:23,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T03:58:23,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T03:58:23,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T03:58:23,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T03:58:23,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T03:58:23,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T03:58:23,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T03:58:23,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T03:58:23,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/nfs.dump.dir in system properties and HBase conf 2024-12-08T03:58:23,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/java.io.tmpdir in system properties and HBase conf 2024-12-08T03:58:23,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T03:58:23,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T03:58:23,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T03:58:24,816 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T03:58:24,904 INFO [Time-limited test {}] log.Log(170): Logging initialized @2142ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T03:58:24,983 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T03:58:25,040 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T03:58:25,059 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T03:58:25,059 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T03:58:25,061 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T03:58:25,076 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T03:58:25,079 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/hadoop.log.dir/,AVAILABLE} 2024-12-08T03:58:25,080 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T03:58:25,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/java.io.tmpdir/jetty-localhost-41233-hadoop-hdfs-3_4_1-tests_jar-_-any-12853011115713349683/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T03:58:25,247 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41233} 2024-12-08T03:58:25,247 INFO [Time-limited test {}] server.Server(415): Started @2486ms 2024-12-08T03:58:25,750 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T03:58:25,758 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T03:58:25,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T03:58:25,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T03:58:25,759 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T03:58:25,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/hadoop.log.dir/,AVAILABLE} 2024-12-08T03:58:25,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T03:58:25,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/java.io.tmpdir/jetty-localhost-36615-hadoop-hdfs-3_4_1-tests_jar-_-any-17900217020389130066/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T03:58:25,856 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36615} 2024-12-08T03:58:25,857 INFO [Time-limited test {}] server.Server(415): Started @3095ms 2024-12-08T03:58:25,901 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T03:58:26,718 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/data/data2/current/BP-1961173970-172.17.0.2-1733630304378/current, will proceed with Du for space computation calculation, 2024-12-08T03:58:26,718 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/data/data1/current/BP-1961173970-172.17.0.2-1733630304378/current, will proceed with Du for space computation calculation, 2024-12-08T03:58:26,749 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T03:58:26,796 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe28f440164d7ab0e with lease ID 0xc970a50e17b02c98: Processing first storage report for DS-cfd005dc-9c73-4850-bfd6-b4c9496e6995 from datanode DatanodeRegistration(127.0.0.1:37085, datanodeUuid=3a01745f-8c30-4451-ba15-ac95fbb8d88b, infoPort=43589, infoSecurePort=0, ipcPort=42739, storageInfo=lv=-57;cid=testClusterID;nsid=1440231594;c=1733630304378) 2024-12-08T03:58:26,797 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe28f440164d7ab0e with lease ID 0xc970a50e17b02c98: from storage DS-cfd005dc-9c73-4850-bfd6-b4c9496e6995 node DatanodeRegistration(127.0.0.1:37085, datanodeUuid=3a01745f-8c30-4451-ba15-ac95fbb8d88b, infoPort=43589, infoSecurePort=0, ipcPort=42739, storageInfo=lv=-57;cid=testClusterID;nsid=1440231594;c=1733630304378), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T03:58:26,798 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe28f440164d7ab0e with lease ID 0xc970a50e17b02c98: Processing first storage report for DS-f34985c5-5a59-4809-95a4-e7fbfd817169 from datanode DatanodeRegistration(127.0.0.1:37085, datanodeUuid=3a01745f-8c30-4451-ba15-ac95fbb8d88b, infoPort=43589, infoSecurePort=0, ipcPort=42739, storageInfo=lv=-57;cid=testClusterID;nsid=1440231594;c=1733630304378) 2024-12-08T03:58:26,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe28f440164d7ab0e with lease ID 0xc970a50e17b02c98: from storage DS-f34985c5-5a59-4809-95a4-e7fbfd817169 node DatanodeRegistration(127.0.0.1:37085, datanodeUuid=3a01745f-8c30-4451-ba15-ac95fbb8d88b, infoPort=43589, infoSecurePort=0, ipcPort=42739, storageInfo=lv=-57;cid=testClusterID;nsid=1440231594;c=1733630304378), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T03:58:26,892 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9 2024-12-08T03:58:26,952 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/zookeeper_0, clientPort=55651, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T03:58:26,960 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55651 2024-12-08T03:58:26,971 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:26,974 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:27,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741825_1001 (size=7) 2024-12-08T03:58:27,589 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 with version=8 2024-12-08T03:58:27,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/hbase-staging 2024-12-08T03:58:27,665 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T03:58:27,908 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T03:58:27,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:58:27,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T03:58:27,921 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T03:58:27,921 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:58:27,921 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T03:58:28,046 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T03:58:28,101 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T03:58:28,110 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T03:58:28,114 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T03:58:28,135 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 102946 (auto-detected) 2024-12-08T03:58:28,136 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T03:58:28,152 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38065 2024-12-08T03:58:28,172 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38065 connecting to ZooKeeper ensemble=127.0.0.1:55651 2024-12-08T03:58:28,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380650x0, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T03:58:28,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38065-0x10003a0e9140000 connected 2024-12-08T03:58:28,378 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:28,381 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:28,394 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T03:58:28,397 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384, hbase.cluster.distributed=false 2024-12-08T03:58:28,416 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T03:58:28,420 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38065 2024-12-08T03:58:28,420 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38065 2024-12-08T03:58:28,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38065 2024-12-08T03:58:28,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38065 2024-12-08T03:58:28,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38065 2024-12-08T03:58:28,510 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T03:58:28,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:58:28,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T03:58:28,513 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T03:58:28,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T03:58:28,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T03:58:28,515 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T03:58:28,517 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T03:58:28,518 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39919 2024-12-08T03:58:28,520 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39919 connecting to ZooKeeper ensemble=127.0.0.1:55651 2024-12-08T03:58:28,521 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:28,525 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:28,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399190x0, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T03:58:28,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399190x0, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T03:58:28,551 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39919-0x10003a0e9140001 connected 2024-12-08T03:58:28,555 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T03:58:28,561 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T03:58:28,564 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T03:58:28,568 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T03:58:28,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39919 2024-12-08T03:58:28,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39919 2024-12-08T03:58:28,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39919 2024-12-08T03:58:28,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39919 2024-12-08T03:58:28,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39919 2024-12-08T03:58:28,585 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32df4b4bb439:38065 2024-12-08T03:58:28,586 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32df4b4bb439,38065,1733630307764 2024-12-08T03:58:28,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:58:28,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:58:28,603 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32df4b4bb439,38065,1733630307764 2024-12-08T03:58:28,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T03:58:28,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:28,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:28,634 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T03:58:28,636 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32df4b4bb439,38065,1733630307764 from backup master directory 2024-12-08T03:58:28,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32df4b4bb439,38065,1733630307764 2024-12-08T03:58:28,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:58:28,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T03:58:28,642 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T03:58:28,643 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32df4b4bb439,38065,1733630307764 2024-12-08T03:58:28,644 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T03:58:28,645 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T03:58:28,701 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/hbase.id] with ID: f9365ce4-250d-489c-a0fa-5ace31918096 2024-12-08T03:58:28,701 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/.tmp/hbase.id 2024-12-08T03:58:28,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741826_1002 (size=42) 2024-12-08T03:58:29,116 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/.tmp/hbase.id]:[hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/hbase.id] 2024-12-08T03:58:29,167 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T03:58:29,171 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T03:58:29,188 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-08T03:58:29,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:29,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:29,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741827_1003 (size=196) 2024-12-08T03:58:29,672 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:58:29,674 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T03:58:29,687 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:29,690 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T03:58:29,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741828_1004 (size=1189) 2024-12-08T03:58:30,139 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store 2024-12-08T03:58:30,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741829_1005 (size=34) 2024-12-08T03:58:30,564 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T03:58:30,569 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:58:30,570 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T03:58:30,571 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:58:30,571 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:58:30,574 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T03:58:30,574 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:58:30,574 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T03:58:30,576 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733630310570Disabling compacts and flushes for region at 1733630310570Disabling writes for close at 1733630310574 (+4 ms)Writing region close event to WAL at 1733630310574Closed at 1733630310574 2024-12-08T03:58:30,578 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/.initializing 2024-12-08T03:58:30,578 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/WALs/32df4b4bb439,38065,1733630307764 2024-12-08T03:58:30,585 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T03:58:30,600 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C38065%2C1733630307764, suffix=, logDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/WALs/32df4b4bb439,38065,1733630307764, archiveDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/oldWALs, maxLogs=10 2024-12-08T03:58:30,623 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/WALs/32df4b4bb439,38065,1733630307764/32df4b4bb439%2C38065%2C1733630307764.1733630310604, exclude list is [], retry=0 2024-12-08T03:58:30,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37085,DS-cfd005dc-9c73-4850-bfd6-b4c9496e6995,DISK] 2024-12-08T03:58:30,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T03:58:30,671 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/WALs/32df4b4bb439,38065,1733630307764/32df4b4bb439%2C38065%2C1733630307764.1733630310604 2024-12-08T03:58:30,671 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43589:43589)] 2024-12-08T03:58:30,672 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:58:30,672 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:58:30,675 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,676 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T03:58:30,733 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:30,735 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:30,735 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T03:58:30,739 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:30,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:58:30,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T03:58:30,743 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:30,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:58:30,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T03:58:30,747 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:30,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:58:30,749 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,752 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,753 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,758 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,759 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,763 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T03:58:30,767 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T03:58:30,771 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:58:30,772 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69707650, jitterRate=0.038724929094314575}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T03:58:30,777 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733630310686Initializing all the Stores at 1733630310688 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630310688Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630310689 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630310689Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630310689Cleaning up temporary data from old regions at 1733630310759 (+70 ms)Region opened successfully at 1733630310776 (+17 ms) 2024-12-08T03:58:30,778 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T03:58:30,805 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7bb616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T03:58:30,831 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T03:58:30,840 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T03:58:30,840 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T03:58:30,843 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T03:58:30,844 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T03:58:30,848 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-08T03:58:30,848 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T03:58:30,877 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T03:58:30,887 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T03:58:30,942 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T03:58:30,947 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T03:58:30,949 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T03:58:30,958 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T03:58:30,960 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T03:58:30,964 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T03:58:30,975 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T03:58:30,976 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T03:58:30,983 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T03:58:31,001 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T03:58:31,008 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T03:58:31,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T03:58:31,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T03:58:31,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:31,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:31,020 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32df4b4bb439,38065,1733630307764, sessionid=0x10003a0e9140000, setting cluster-up flag (Was=false) 2024-12-08T03:58:31,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:31,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:31,075 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T03:58:31,078 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,38065,1733630307764 2024-12-08T03:58:31,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:31,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:31,125 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T03:58:31,128 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,38065,1733630307764 2024-12-08T03:58:31,138 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T03:58:31,177 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(746): ClusterId : f9365ce4-250d-489c-a0fa-5ace31918096 2024-12-08T03:58:31,180 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T03:58:31,204 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T03:58:31,209 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T03:58:31,209 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T03:58:31,212 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T03:58:31,218 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T03:58:31,218 DEBUG [RS:0;32df4b4bb439:39919 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a255822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T03:58:31,219 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T03:58:31,225 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32df4b4bb439,38065,1733630307764 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T03:58:31,233 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32df4b4bb439:39919 2024-12-08T03:58:31,235 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:58:31,235 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:58:31,235 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:58:31,235 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T03:58:31,235 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T03:58:31,235 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T03:58:31,235 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T03:58:31,235 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32df4b4bb439:0, corePoolSize=10, maxPoolSize=10 2024-12-08T03:58:31,236 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,236 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T03:58:31,236 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,238 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,38065,1733630307764 with port=39919, startcode=1733630308479 2024-12-08T03:58:31,240 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733630341240 2024-12-08T03:58:31,242 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T03:58:31,242 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T03:58:31,243 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T03:58:31,244 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T03:58:31,247 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:31,247 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T03:58:31,248 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T03:58:31,248 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T03:58:31,248 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T03:58:31,248 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T03:58:31,249 DEBUG [RS:0;32df4b4bb439:39919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T03:58:31,249 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,255 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T03:58:31,257 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T03:58:31,257 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T03:58:31,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741831_1007 (size=1321) 2024-12-08T03:58:31,260 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T03:58:31,260 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T03:58:31,263 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630311261,5,FailOnTimeoutGroup] 2024-12-08T03:58:31,264 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630311263,5,FailOnTimeoutGroup] 2024-12-08T03:58:31,264 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,264 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T03:58:31,265 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,265 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38943, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T03:58:31,288 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38065 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,39919,1733630308479 2024-12-08T03:58:31,290 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38065 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:31,302 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T03:58:31,302 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37247 2024-12-08T03:58:31,302 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T03:58:31,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T03:58:31,334 DEBUG [RS:0;32df4b4bb439:39919 {}] zookeeper.ZKUtil(111): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,39919,1733630308479 2024-12-08T03:58:31,334 WARN [RS:0;32df4b4bb439:39919 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T03:58:31,334 INFO [RS:0;32df4b4bb439:39919 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T03:58:31,335 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479 2024-12-08T03:58:31,337 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,39919,1733630308479] 2024-12-08T03:58:31,360 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T03:58:31,372 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T03:58:31,377 INFO [RS:0;32df4b4bb439:39919 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T03:58:31,377 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,378 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T03:58:31,383 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T03:58:31,384 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,384 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,384 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,384 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,384 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,385 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,386 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T03:58:31,386 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T03:58:31,386 DEBUG [RS:0;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T03:58:31,387 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,388 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,388 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,388 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,388 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,388 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,39919,1733630308479-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T03:58:31,404 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T03:58:31,406 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,39919,1733630308479-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,406 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,406 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.Replication(171): 32df4b4bb439,39919,1733630308479 started 2024-12-08T03:58:31,423 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:31,423 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,39919,1733630308479, RpcServer on 32df4b4bb439/172.17.0.2:39919, sessionid=0x10003a0e9140001 2024-12-08T03:58:31,424 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T03:58:31,424 DEBUG [RS:0;32df4b4bb439:39919 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,39919,1733630308479 2024-12-08T03:58:31,424 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,39919,1733630308479' 2024-12-08T03:58:31,424 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T03:58:31,425 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T03:58:31,426 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T03:58:31,426 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T03:58:31,427 DEBUG [RS:0;32df4b4bb439:39919 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,39919,1733630308479 2024-12-08T03:58:31,427 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,39919,1733630308479' 2024-12-08T03:58:31,427 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T03:58:31,428 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T03:58:31,428 DEBUG [RS:0;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T03:58:31,428 INFO [RS:0;32df4b4bb439:39919 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T03:58:31,429 INFO [RS:0;32df4b4bb439:39919 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T03:58:31,538 INFO [RS:0;32df4b4bb439:39919 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T03:58:31,542 INFO [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C39919%2C1733630308479, suffix=, logDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479, archiveDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/oldWALs, maxLogs=32 2024-12-08T03:58:31,557 DEBUG [RS:0;32df4b4bb439:39919 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479/32df4b4bb439%2C39919%2C1733630308479.1733630311545, exclude list is [], retry=0 2024-12-08T03:58:31,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37085,DS-cfd005dc-9c73-4850-bfd6-b4c9496e6995,DISK] 2024-12-08T03:58:31,565 INFO [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479/32df4b4bb439%2C39919%2C1733630308479.1733630311545 2024-12-08T03:58:31,566 DEBUG [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43589:43589)] 2024-12-08T03:58:31,662 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T03:58:31,664 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T03:58:31,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741833_1009 (size=32) 2024-12-08T03:58:32,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:58:32,090 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T03:58:32,092 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T03:58:32,093 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T03:58:32,096 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T03:58:32,096 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T03:58:32,100 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T03:58:32,100 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,101 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T03:58:32,104 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T03:58:32,104 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,106 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T03:58:32,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740 2024-12-08T03:58:32,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740 2024-12-08T03:58:32,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T03:58:32,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T03:58:32,111 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:58:32,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T03:58:32,117 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:58:32,118 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72682565, jitterRate=0.08305461704730988}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:58:32,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733630312085Initializing all the Stores at 1733630312089 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630312089Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630312089Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630312089Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630312089Cleaning up temporary data from old regions at 1733630312110 (+21 ms)Region opened successfully at 1733630312119 (+9 ms) 2024-12-08T03:58:32,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T03:58:32,120 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T03:58:32,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T03:58:32,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T03:58:32,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T03:58:32,121 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T03:58:32,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630312120Disabling compacts and flushes for region at 1733630312120Disabling writes for close at 1733630312120Writing region close event to WAL at 1733630312121 (+1 ms)Closed at 1733630312121 2024-12-08T03:58:32,124 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T03:58:32,124 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T03:58:32,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T03:58:32,136 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T03:58:32,139 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T03:58:32,298 DEBUG [32df4b4bb439:38065 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T03:58:32,309 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:32,313 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,39919,1733630308479, state=OPENING 2024-12-08T03:58:32,340 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T03:58:32,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:32,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T03:58:32,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:58:32,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:58:32,353 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T03:58:32,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:58:32,540 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T03:58:32,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T03:58:32,555 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T03:58:32,555 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T03:58:32,556 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T03:58:32,558 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C39919%2C1733630308479.meta, suffix=.meta, logDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479, archiveDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/oldWALs, maxLogs=32 2024-12-08T03:58:32,571 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479/32df4b4bb439%2C39919%2C1733630308479.meta.1733630312560.meta, exclude list is [], retry=0 2024-12-08T03:58:32,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37085,DS-cfd005dc-9c73-4850-bfd6-b4c9496e6995,DISK] 2024-12-08T03:58:32,577 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/WALs/32df4b4bb439,39919,1733630308479/32df4b4bb439%2C39919%2C1733630308479.meta.1733630312560.meta 2024-12-08T03:58:32,578 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43589:43589)] 2024-12-08T03:58:32,578 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:58:32,580 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T03:58:32,582 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T03:58:32,586 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T03:58:32,590 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T03:58:32,590 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:58:32,590 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T03:58:32,590 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T03:58:32,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T03:58:32,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T03:58:32,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T03:58:32,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T03:58:32,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T03:58:32,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T03:58:32,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T03:58:32,602 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T03:58:32,602 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:32,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T03:58:32,603 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T03:58:32,605 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740 2024-12-08T03:58:32,608 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740 2024-12-08T03:58:32,610 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T03:58:32,610 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T03:58:32,611 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:58:32,614 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T03:58:32,616 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71104024, jitterRate=0.0595325231552124}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:58:32,616 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T03:58:32,617 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733630312591Writing region info on filesystem at 1733630312591Initializing all the Stores at 1733630312593 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630312593Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630312593Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630312593Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630312593Cleaning up temporary data from old regions at 1733630312610 (+17 ms)Running coprocessor post-open hooks at 1733630312616 (+6 ms)Region opened successfully at 1733630312617 (+1 ms) 2024-12-08T03:58:32,623 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733630312531 2024-12-08T03:58:32,633 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T03:58:32,634 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T03:58:32,635 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:32,637 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,39919,1733630308479, state=OPEN 2024-12-08T03:58:32,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T03:58:32,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T03:58:32,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:58:32,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T03:58:32,693 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:32,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T03:58:32,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,39919,1733630308479 in 336 msec 2024-12-08T03:58:32,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T03:58:32,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 571 msec 2024-12-08T03:58:32,708 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T03:58:32,708 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T03:58:32,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:32,726 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:32,747 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:32,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57033, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:32,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6000 sec 2024-12-08T03:58:32,767 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733630312767, completionTime=-1 2024-12-08T03:58:32,770 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T03:58:32,770 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T03:58:32,791 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T03:58:32,792 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733630372792 2024-12-08T03:58:32,792 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733630432792 2024-12-08T03:58:32,792 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-08T03:58:32,794 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,38065,1733630307764-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,795 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,38065,1733630307764-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,795 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,38065,1733630307764-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,796 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32df4b4bb439:38065, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,797 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,797 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,803 DEBUG [master/32df4b4bb439:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T03:58:32,824 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.180sec 2024-12-08T03:58:32,825 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T03:58:32,826 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T03:58:32,827 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T03:58:32,828 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T03:58:32,828 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T03:58:32,829 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,38065,1733630307764-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T03:58:32,829 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,38065,1733630307764-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T03:58:32,839 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T03:58:32,840 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T03:58:32,840 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,38065,1733630307764-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T03:58:32,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:32,891 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T03:58:32,892 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T03:58:32,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:32,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:32,902 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:32,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:32,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:32,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:32,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:32,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:32,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:32,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32df4b4bb439,38065,1733630307764 2024-12-08T03:58:32,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=382, ProcessCount=11, AvailableMemoryMB=17165 2024-12-08T03:58:32,962 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:58:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T03:58:32,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-12-08T03:58:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:58:32,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:58:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:58:32,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:58:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:58:33,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:58:33,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-08T03:58:33,014 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:33,016 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:58:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:58:33,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741835_1011 (size=1061) 2024-12-08T03:58:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:58:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:58:33,435 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T03:58:33,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741836_1012 (size=53) 2024-12-08T03:58:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:58:33,850 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:58:33,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 97ee24ee195dd7bff50b82d61598e8ac, disabling compactions & flushes 2024-12-08T03:58:33,851 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:33,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:33,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. after waiting 0 ms 2024-12-08T03:58:33,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:33,851 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:33,852 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 97ee24ee195dd7bff50b82d61598e8ac: Waiting for close lock at 1733630313850Disabling compacts and flushes for region at 1733630313850Disabling writes for close at 1733630313851 (+1 ms)Writing region close event to WAL at 1733630313851Closed at 1733630313851 2024-12-08T03:58:33,857 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:58:33,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733630313857"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630313857"}]},"ts":"1733630313857"} 2024-12-08T03:58:33,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:58:33,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:58:33,872 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630313870"}]},"ts":"1733630313870"} 2024-12-08T03:58:33,876 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:58:33,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, ASSIGN}] 2024-12-08T03:58:33,880 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, ASSIGN 2024-12-08T03:58:33,882 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, ASSIGN; state=OFFLINE, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=false 2024-12-08T03:58:34,034 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=97ee24ee195dd7bff50b82d61598e8ac, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:34,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, ASSIGN because future has completed 2024-12-08T03:58:34,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:58:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:58:34,206 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:34,207 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:58:34,207 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,208 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:58:34,208 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,208 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,211 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,216 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:58:34,217 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97ee24ee195dd7bff50b82d61598e8ac columnFamilyName A 2024-12-08T03:58:34,217 DEBUG [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:34,218 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.HStore(327): Store=97ee24ee195dd7bff50b82d61598e8ac/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:58:34,218 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,221 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:58:34,221 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97ee24ee195dd7bff50b82d61598e8ac columnFamilyName B 2024-12-08T03:58:34,221 DEBUG [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:34,222 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.HStore(327): Store=97ee24ee195dd7bff50b82d61598e8ac/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:58:34,223 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,225 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:58:34,226 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97ee24ee195dd7bff50b82d61598e8ac columnFamilyName C 2024-12-08T03:58:34,226 DEBUG [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:58:34,227 INFO [StoreOpener-97ee24ee195dd7bff50b82d61598e8ac-1 {}] regionserver.HStore(327): Store=97ee24ee195dd7bff50b82d61598e8ac/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:58:34,227 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:34,228 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,229 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,230 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,233 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,233 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,234 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:58:34,236 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,240 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:58:34,241 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 97ee24ee195dd7bff50b82d61598e8ac; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60689881, jitterRate=-0.09565030038356781}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:58:34,241 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:34,243 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 97ee24ee195dd7bff50b82d61598e8ac: Running coprocessor pre-open hook at 1733630314208Writing region info on filesystem at 1733630314208Initializing all the Stores at 1733630314211 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630314211Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630314211Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630314211Cleaning up temporary data from old regions at 1733630314233 (+22 ms)Running coprocessor post-open hooks at 1733630314241 (+8 ms)Region opened successfully at 1733630314243 (+2 ms) 2024-12-08T03:58:34,245 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., pid=6, masterSystemTime=1733630314197 2024-12-08T03:58:34,248 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:34,249 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:34,250 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=97ee24ee195dd7bff50b82d61598e8ac, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:34,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:58:34,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T03:58:34,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 in 214 msec 2024-12-08T03:58:34,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T03:58:34,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, ASSIGN in 383 msec 2024-12-08T03:58:34,266 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:58:34,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630314267"}]},"ts":"1733630314267"} 2024-12-08T03:58:34,269 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:58:34,271 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:58:34,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2730 sec 2024-12-08T03:58:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T03:58:35,169 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:35,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cdc6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,183 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1806c24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,192 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:58:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T03:58:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78210a9b 2024-12-08T03:58:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:58:35,198 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:58:35,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,204 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,210 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,218 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,221 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,225 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,229 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5970e36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,234 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,237 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,241 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,244 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dae11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,247 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20584f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,250 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7aa9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,253 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aaa9530, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:58:35,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:58:35,259 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:58:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:58:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:58:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:58:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:58:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:35,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:58:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:58:35,267 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,267 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,267 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,267 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,267 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,268 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,269 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,271 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,272 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:58:35,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:58:35,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:58:35,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:35,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:58:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-08T03:58:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,295 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:58:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:58:35,297 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:35,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:35,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:58:35,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:35,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:35,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:35,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:35,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:35,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:58:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630375452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630375448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630375465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:58:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630375464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630375472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:35,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:35,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/bf36a2ada2b9487d9be511bc95a60252 is 50, key is test_row_0/A:col10/1733630315351/Put/seqid=0 2024-12-08T03:58:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741837_1013 (size=16681) 2024-12-08T03:58:35,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/bf36a2ada2b9487d9be511bc95a60252 2024-12-08T03:58:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:58:35,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/b63f746f735143d9aff527e13670d16b is 50, key is test_row_0/B:col10/1733630315351/Put/seqid=0 2024-12-08T03:58:35,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630375618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630375618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630375619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630375620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630375620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741838_1014 (size=12001) 2024-12-08T03:58:35,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:58:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:35,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:58:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:35,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630375839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630375839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630375840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630375840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630375842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:58:35,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:58:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:36,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:36,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/b63f746f735143d9aff527e13670d16b 2024-12-08T03:58:36,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/bc0803c5eedd49058c0fb1a3eec9765b is 50, key is test_row_0/C:col10/1733630315351/Put/seqid=0 2024-12-08T03:58:36,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741839_1015 (size=12001) 2024-12-08T03:58:36,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/bc0803c5eedd49058c0fb1a3eec9765b 2024-12-08T03:58:36,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:58:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:36,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:36,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:36,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630376158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630376160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630376159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630376160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630376164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/bf36a2ada2b9487d9be511bc95a60252 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/bf36a2ada2b9487d9be511bc95a60252 2024-12-08T03:58:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/bf36a2ada2b9487d9be511bc95a60252, entries=250, sequenceid=14, filesize=16.3 K 2024-12-08T03:58:36,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/b63f746f735143d9aff527e13670d16b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/b63f746f735143d9aff527e13670d16b 2024-12-08T03:58:36,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/b63f746f735143d9aff527e13670d16b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T03:58:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/bc0803c5eedd49058c0fb1a3eec9765b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/bc0803c5eedd49058c0fb1a3eec9765b 2024-12-08T03:58:36,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/bc0803c5eedd49058c0fb1a3eec9765b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T03:58:36,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 97ee24ee195dd7bff50b82d61598e8ac in 889ms, sequenceid=14, compaction requested=false 2024-12-08T03:58:36,246 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:58:36,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:36,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T03:58:36,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:36,315 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T03:58:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:36,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:36,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/2fd39d6bfbbd46bc9077ca288ca70a5e is 50, key is test_row_0/A:col10/1733630315466/Put/seqid=0 2024-12-08T03:58:36,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741840_1016 (size=9657) 2024-12-08T03:58:36,351 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/2fd39d6bfbbd46bc9077ca288ca70a5e 2024-12-08T03:58:36,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/32758d1e325846b4a7d5ee36663ea904 is 50, key is test_row_0/B:col10/1733630315466/Put/seqid=0 2024-12-08T03:58:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741841_1017 (size=9657) 2024-12-08T03:58:36,396 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/32758d1e325846b4a7d5ee36663ea904 2024-12-08T03:58:36,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/253dc4e95322439ea219093718d6195d is 50, key is test_row_0/C:col10/1733630315466/Put/seqid=0 2024-12-08T03:58:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:58:36,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741842_1018 (size=9657) 2024-12-08T03:58:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:36,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630376695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630376702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630376703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630376706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630376705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630376817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630376818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630376818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630376818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630376818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:36,855 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/253dc4e95322439ea219093718d6195d 2024-12-08T03:58:36,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/2fd39d6bfbbd46bc9077ca288ca70a5e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/2fd39d6bfbbd46bc9077ca288ca70a5e 2024-12-08T03:58:36,888 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/2fd39d6bfbbd46bc9077ca288ca70a5e, entries=100, sequenceid=33, filesize=9.4 K 2024-12-08T03:58:36,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/32758d1e325846b4a7d5ee36663ea904 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/32758d1e325846b4a7d5ee36663ea904 2024-12-08T03:58:36,913 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/32758d1e325846b4a7d5ee36663ea904, entries=100, sequenceid=33, filesize=9.4 K 2024-12-08T03:58:36,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/253dc4e95322439ea219093718d6195d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/253dc4e95322439ea219093718d6195d 2024-12-08T03:58:36,934 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/253dc4e95322439ea219093718d6195d, entries=100, sequenceid=33, filesize=9.4 K 2024-12-08T03:58:36,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-08T03:58:36,936 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 97ee24ee195dd7bff50b82d61598e8ac in 622ms, sequenceid=33, compaction requested=false 2024-12-08T03:58:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-08T03:58:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-08T03:58:36,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T03:58:36,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-12-08T03:58:36,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6710 sec 2024-12-08T03:58:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:37,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T03:58:37,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:37,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:37,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:37,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/4fbacb4393ce4404b0404bc13a87db18 is 50, key is test_row_0/A:col10/1733630316702/Put/seqid=0 2024-12-08T03:58:37,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630377064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630377067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630377071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630377068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630377074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741843_1019 (size=12001) 2024-12-08T03:58:37,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/4fbacb4393ce4404b0404bc13a87db18 2024-12-08T03:58:37,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/11b9d6cb63724d3ea55369184cc5cb66 is 50, key is test_row_0/B:col10/1733630316702/Put/seqid=0 2024-12-08T03:58:37,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741844_1020 (size=12001) 2024-12-08T03:58:37,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/11b9d6cb63724d3ea55369184cc5cb66 2024-12-08T03:58:37,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/f7e106432b2841d6910e6629d416d437 is 50, key is test_row_0/C:col10/1733630316702/Put/seqid=0 2024-12-08T03:58:37,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630377188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630377189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630377188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630377189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630377200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741845_1021 (size=12001) 2024-12-08T03:58:37,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/f7e106432b2841d6910e6629d416d437 2024-12-08T03:58:37,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/4fbacb4393ce4404b0404bc13a87db18 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4fbacb4393ce4404b0404bc13a87db18 2024-12-08T03:58:37,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4fbacb4393ce4404b0404bc13a87db18, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T03:58:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/11b9d6cb63724d3ea55369184cc5cb66 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/11b9d6cb63724d3ea55369184cc5cb66 2024-12-08T03:58:37,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/11b9d6cb63724d3ea55369184cc5cb66, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T03:58:37,268 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-08T03:58:37,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/f7e106432b2841d6910e6629d416d437 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f7e106432b2841d6910e6629d416d437 2024-12-08T03:58:37,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f7e106432b2841d6910e6629d416d437, entries=150, sequenceid=49, filesize=11.7 K 2024-12-08T03:58:37,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 97ee24ee195dd7bff50b82d61598e8ac in 260ms, sequenceid=49, compaction requested=true 2024-12-08T03:58:37,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:37,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:37,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:37,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:37,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:37,302 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:37,303 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:37,305 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:37,306 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:37,306 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/b63f746f735143d9aff527e13670d16b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/32758d1e325846b4a7d5ee36663ea904, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/11b9d6cb63724d3ea55369184cc5cb66] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=32.9 K 2024-12-08T03:58:37,308 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b63f746f735143d9aff527e13670d16b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733630315345 2024-12-08T03:58:37,309 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 32758d1e325846b4a7d5ee36663ea904, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733630315440 2024-12-08T03:58:37,309 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:37,310 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:37,310 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:37,310 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/bf36a2ada2b9487d9be511bc95a60252, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/2fd39d6bfbbd46bc9077ca288ca70a5e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4fbacb4393ce4404b0404bc13a87db18] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=37.4 K 2024-12-08T03:58:37,310 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b9d6cb63724d3ea55369184cc5cb66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733630316702 2024-12-08T03:58:37,311 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf36a2ada2b9487d9be511bc95a60252, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733630315313 2024-12-08T03:58:37,313 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fd39d6bfbbd46bc9077ca288ca70a5e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733630315440 2024-12-08T03:58:37,314 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbacb4393ce4404b0404bc13a87db18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733630316702 2024-12-08T03:58:37,347 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#9 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:37,348 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/50a35c1eea8d4ee990c939a20268e2a8 is 50, key is test_row_0/B:col10/1733630316702/Put/seqid=0 2024-12-08T03:58:37,355 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T03:58:37,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741846_1022 (size=12104) 2024-12-08T03:58:37,371 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#10 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:37,372 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/6fcc2d58eb924ea6a6a65d0fad42c01e is 50, key is test_row_0/A:col10/1733630316702/Put/seqid=0 2024-12-08T03:58:37,393 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/50a35c1eea8d4ee990c939a20268e2a8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/50a35c1eea8d4ee990c939a20268e2a8 2024-12-08T03:58:37,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741847_1023 (size=12104) 2024-12-08T03:58:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:37,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T03:58:37,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:37,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:37,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:37,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/4beb5da98d464c0daf569ce6becafbe5 is 50, key is test_row_0/A:col10/1733630317070/Put/seqid=0 2024-12-08T03:58:37,425 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into 50a35c1eea8d4ee990c939a20268e2a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:37,425 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:37,428 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630317298; duration=0sec 2024-12-08T03:58:37,429 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:37,429 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:37,430 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:37,433 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:37,433 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:37,433 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:37,433 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/bc0803c5eedd49058c0fb1a3eec9765b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/253dc4e95322439ea219093718d6195d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f7e106432b2841d6910e6629d416d437] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=32.9 K 2024-12-08T03:58:37,436 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting bc0803c5eedd49058c0fb1a3eec9765b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733630315345 2024-12-08T03:58:37,437 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 253dc4e95322439ea219093718d6195d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733630315440 2024-12-08T03:58:37,439 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e106432b2841d6910e6629d416d437, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733630316702 2024-12-08T03:58:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T03:58:37,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:37,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-08T03:58:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:58:37,453 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:37,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630377440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,456 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:37,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:37,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630377444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630377440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630377452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630377448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741848_1024 (size=16681) 2024-12-08T03:58:37,476 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#12 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:37,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/4beb5da98d464c0daf569ce6becafbe5 2024-12-08T03:58:37,477 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6c33171b7e1d4ee2a6ebcd333951b33f is 50, key is test_row_0/C:col10/1733630316702/Put/seqid=0 2024-12-08T03:58:37,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6d6e137826014dc19c50fa86f0de4c88 is 50, key is test_row_0/B:col10/1733630317070/Put/seqid=0 2024-12-08T03:58:37,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741849_1025 (size=12104) 2024-12-08T03:58:37,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741850_1026 (size=12001) 2024-12-08T03:58:37,530 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6c33171b7e1d4ee2a6ebcd333951b33f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6c33171b7e1d4ee2a6ebcd333951b33f 2024-12-08T03:58:37,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6d6e137826014dc19c50fa86f0de4c88 2024-12-08T03:58:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:58:37,558 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 6c33171b7e1d4ee2a6ebcd333951b33f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:37,559 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:37,559 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630317299; duration=0sec 2024-12-08T03:58:37,559 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:37,559 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630377558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0bb7639f856e45678fcfc1e14f586fff is 50, key is test_row_0/C:col10/1733630317070/Put/seqid=0 2024-12-08T03:58:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630377567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630377568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741851_1027 (size=12001) 2024-12-08T03:58:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630377568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630377568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0bb7639f856e45678fcfc1e14f586fff 2024-12-08T03:58:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/4beb5da98d464c0daf569ce6becafbe5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4beb5da98d464c0daf569ce6becafbe5 2024-12-08T03:58:37,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4beb5da98d464c0daf569ce6becafbe5, entries=250, sequenceid=68, filesize=16.3 K 2024-12-08T03:58:37,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T03:58:37,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6d6e137826014dc19c50fa86f0de4c88 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6d6e137826014dc19c50fa86f0de4c88 2024-12-08T03:58:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:37,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:37,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:37,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:37,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6d6e137826014dc19c50fa86f0de4c88, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T03:58:37,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0bb7639f856e45678fcfc1e14f586fff as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0bb7639f856e45678fcfc1e14f586fff 2024-12-08T03:58:37,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0bb7639f856e45678fcfc1e14f586fff, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T03:58:37,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 97ee24ee195dd7bff50b82d61598e8ac in 249ms, sequenceid=68, compaction requested=false 2024-12-08T03:58:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:58:37,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T03:58:37,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:37,769 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T03:58:37,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:37,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:37,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/835d4f2caf8b4af4965f5db13a56b496 is 50, key is test_row_0/A:col10/1733630317447/Put/seqid=0 2024-12-08T03:58:37,829 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/6fcc2d58eb924ea6a6a65d0fad42c01e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6fcc2d58eb924ea6a6a65d0fad42c01e 2024-12-08T03:58:37,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741852_1028 (size=12001) 2024-12-08T03:58:37,855 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into 6fcc2d58eb924ea6a6a65d0fad42c01e(size=11.8 K), total size for store is 28.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:37,855 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:37,855 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630317295; duration=0sec 2024-12-08T03:58:37,855 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:37,856 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630377843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630377843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630377853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630377857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630377857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630377967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630377967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630377968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630377968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:37,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630377978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:58:38,103 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:58:38,167 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T03:58:38,167 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T03:58:38,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T03:58:38,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T03:58:38,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T03:58:38,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T03:58:38,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T03:58:38,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T03:58:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630378177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630378178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630378178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630378178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630378188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,252 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/835d4f2caf8b4af4965f5db13a56b496 2024-12-08T03:58:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/242aa16b88c043e3b3d7a50df1b27619 is 50, key is test_row_0/B:col10/1733630317447/Put/seqid=0 2024-12-08T03:58:38,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741853_1029 (size=12001) 2024-12-08T03:58:38,284 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/242aa16b88c043e3b3d7a50df1b27619 2024-12-08T03:58:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/41c8cff045d24bb7bacf361ac843e36e is 50, key is test_row_0/C:col10/1733630317447/Put/seqid=0 2024-12-08T03:58:38,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741854_1030 (size=12001) 2024-12-08T03:58:38,311 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/41c8cff045d24bb7bacf361ac843e36e 2024-12-08T03:58:38,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/835d4f2caf8b4af4965f5db13a56b496 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/835d4f2caf8b4af4965f5db13a56b496 2024-12-08T03:58:38,343 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/835d4f2caf8b4af4965f5db13a56b496, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T03:58:38,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/242aa16b88c043e3b3d7a50df1b27619 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/242aa16b88c043e3b3d7a50df1b27619 2024-12-08T03:58:38,364 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/242aa16b88c043e3b3d7a50df1b27619, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T03:58:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/41c8cff045d24bb7bacf361ac843e36e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/41c8cff045d24bb7bacf361ac843e36e 2024-12-08T03:58:38,378 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/41c8cff045d24bb7bacf361ac843e36e, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T03:58:38,380 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 97ee24ee195dd7bff50b82d61598e8ac in 611ms, sequenceid=83, compaction requested=true 2024-12-08T03:58:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:38,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:38,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-08T03:58:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-08T03:58:38,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-08T03:58:38,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 927 msec 2024-12-08T03:58:38,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 941 msec 2024-12-08T03:58:38,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630317846, max timestamp=1733630318490Num uniques 859; 2024-12-08T03:58:38,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:38,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630317846, max timestamp=1733630318490Num uniques 859; 2024-12-08T03:58:38,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630317846, max timestamp=1733630318490Num uniques 859; 2024-12-08T03:58:38,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:38,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:38,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-08T03:58:38,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:38,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:38,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/df99fd0072344182bdd28dd809b2cc9c is 50, key is test_row_0/A:col10/1733630318507/Put/seqid=0 2024-12-08T03:58:38,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741855_1031 (size=12001) 2024-12-08T03:58:38,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/df99fd0072344182bdd28dd809b2cc9c 2024-12-08T03:58:38,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/ebeec46810474c8f9bc5611f665d3b30 is 50, key is test_row_0/B:col10/1733630318507/Put/seqid=0 2024-12-08T03:58:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T03:58:38,588 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:38,589 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:38,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630318570, max timestamp=1733630318581Num uniques 859; 2024-12-08T03:58:38,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:58:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-08T03:58:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:58:38,594 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:38,595 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:38,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:38,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630318570, max timestamp=1733630318581Num uniques 859; 2024-12-08T03:58:38,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:58:38,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741856_1032 (size=12001) 2024-12-08T03:58:38,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733630318570, max timestamp=1733630318581Num uniques 859; 2024-12-08T03:58:38,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T03:58:38,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/ebeec46810474c8f9bc5611f665d3b30 2024-12-08T03:58:38,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/4ec9a0fc7a9548b9b6502f1bef72e9c0 is 50, key is test_row_0/C:col10/1733630318507/Put/seqid=0 2024-12-08T03:58:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630378647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630378647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630378650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630378653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741857_1033 (size=12001) 2024-12-08T03:58:38,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630378666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:58:38,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:58:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:38,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:38,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:38,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630378767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630378767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630378778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630378778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630378778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:58:38,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:38,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:58:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630378977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630378978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630378998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630378998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630378998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:58:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:39,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:39,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/4ec9a0fc7a9548b9b6502f1bef72e9c0 2024-12-08T03:58:39,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/df99fd0072344182bdd28dd809b2cc9c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/df99fd0072344182bdd28dd809b2cc9c 2024-12-08T03:58:39,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/df99fd0072344182bdd28dd809b2cc9c, entries=150, sequenceid=109, filesize=11.7 K 2024-12-08T03:58:39,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/ebeec46810474c8f9bc5611f665d3b30 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/ebeec46810474c8f9bc5611f665d3b30 2024-12-08T03:58:39,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/ebeec46810474c8f9bc5611f665d3b30, entries=150, sequenceid=109, filesize=11.7 K 2024-12-08T03:58:39,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/4ec9a0fc7a9548b9b6502f1bef72e9c0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/4ec9a0fc7a9548b9b6502f1bef72e9c0 2024-12-08T03:58:39,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/4ec9a0fc7a9548b9b6502f1bef72e9c0, entries=150, sequenceid=109, filesize=11.7 K 2024-12-08T03:58:39,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=131.55 KB/134705 for 97ee24ee195dd7bff50b82d61598e8ac in 626ms, sequenceid=109, compaction requested=true 2024-12-08T03:58:39,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:39,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:39,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:39,141 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:58:39,141 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:58:39,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:39,147 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:58:39,147 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:58:39,147 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:39,147 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:39,147 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,147 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,147 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/50a35c1eea8d4ee990c939a20268e2a8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6d6e137826014dc19c50fa86f0de4c88, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/242aa16b88c043e3b3d7a50df1b27619, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/ebeec46810474c8f9bc5611f665d3b30] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=47.0 K 2024-12-08T03:58:39,147 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6fcc2d58eb924ea6a6a65d0fad42c01e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4beb5da98d464c0daf569ce6becafbe5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/835d4f2caf8b4af4965f5db13a56b496, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/df99fd0072344182bdd28dd809b2cc9c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=51.5 K 2024-12-08T03:58:39,148 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fcc2d58eb924ea6a6a65d0fad42c01e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733630316702 2024-12-08T03:58:39,148 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a35c1eea8d4ee990c939a20268e2a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733630316702 2024-12-08T03:58:39,149 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4beb5da98d464c0daf569ce6becafbe5, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733630317067 2024-12-08T03:58:39,149 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d6e137826014dc19c50fa86f0de4c88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733630317070 2024-12-08T03:58:39,150 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835d4f2caf8b4af4965f5db13a56b496, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733630317426 2024-12-08T03:58:39,150 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 242aa16b88c043e3b3d7a50df1b27619, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733630317426 2024-12-08T03:58:39,150 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting df99fd0072344182bdd28dd809b2cc9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733630317852 2024-12-08T03:58:39,150 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ebeec46810474c8f9bc5611f665d3b30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733630317852 2024-12-08T03:58:39,174 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:39,175 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/c4107be9646547bd8bd464a4f681a51c is 50, key is test_row_0/A:col10/1733630318507/Put/seqid=0 2024-12-08T03:58:39,182 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:39,182 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/2250060066174d43b4f6001454ec070a is 50, key is test_row_0/B:col10/1733630318507/Put/seqid=0 2024-12-08T03:58:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741858_1034 (size=12241) 2024-12-08T03:58:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741859_1035 (size=12241) 2024-12-08T03:58:39,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T03:58:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-12-08T03:58:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:58:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:39,222 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/2250060066174d43b4f6001454ec070a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2250060066174d43b4f6001454ec070a 2024-12-08T03:58:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/562367154e164ae1ae7982b58d6c62e0 is 50, key is test_row_0/A:col10/1733630318649/Put/seqid=0 2024-12-08T03:58:39,248 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into 2250060066174d43b4f6001454ec070a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:39,248 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:39,248 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=12, startTime=1733630319141; duration=0sec 2024-12-08T03:58:39,249 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:39,249 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:39,249 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:58:39,252 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:58:39,252 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:39,252 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,253 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6c33171b7e1d4ee2a6ebcd333951b33f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0bb7639f856e45678fcfc1e14f586fff, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/41c8cff045d24bb7bacf361ac843e36e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/4ec9a0fc7a9548b9b6502f1bef72e9c0] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=47.0 K 2024-12-08T03:58:39,254 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c33171b7e1d4ee2a6ebcd333951b33f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733630316702 2024-12-08T03:58:39,256 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb7639f856e45678fcfc1e14f586fff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733630317070 2024-12-08T03:58:39,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741860_1036 (size=12151) 2024-12-08T03:58:39,257 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c8cff045d24bb7bacf361ac843e36e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733630317426 2024-12-08T03:58:39,261 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/562367154e164ae1ae7982b58d6c62e0 2024-12-08T03:58:39,262 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec9a0fc7a9548b9b6502f1bef72e9c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733630317852 2024-12-08T03:58:39,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/70ac9ec62b5f483cb5b11248eb84910a is 50, key is test_row_0/B:col10/1733630318649/Put/seqid=0 2024-12-08T03:58:39,297 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:39,301 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/890c81f1afea447982453841cdbc3aa5 is 50, key is test_row_0/C:col10/1733630318507/Put/seqid=0 2024-12-08T03:58:39,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741861_1037 (size=12151) 2024-12-08T03:58:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630379313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,320 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/70ac9ec62b5f483cb5b11248eb84910a 2024-12-08T03:58:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630379316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630379318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741862_1038 (size=12241) 2024-12-08T03:58:39,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630379320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630379323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/981e4bfd5290455d83866100d17d1360 is 50, key is test_row_0/C:col10/1733630318649/Put/seqid=0 2024-12-08T03:58:39,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741863_1039 (size=12151) 2024-12-08T03:58:39,370 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/981e4bfd5290455d83866100d17d1360 2024-12-08T03:58:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/562367154e164ae1ae7982b58d6c62e0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/562367154e164ae1ae7982b58d6c62e0 2024-12-08T03:58:39,397 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/562367154e164ae1ae7982b58d6c62e0, entries=150, sequenceid=145, filesize=11.9 K 2024-12-08T03:58:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/70ac9ec62b5f483cb5b11248eb84910a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/70ac9ec62b5f483cb5b11248eb84910a 2024-12-08T03:58:39,414 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/70ac9ec62b5f483cb5b11248eb84910a, entries=150, sequenceid=145, filesize=11.9 K 2024-12-08T03:58:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/981e4bfd5290455d83866100d17d1360 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/981e4bfd5290455d83866100d17d1360 2024-12-08T03:58:39,426 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/981e4bfd5290455d83866100d17d1360, entries=150, sequenceid=145, filesize=11.9 K 2024-12-08T03:58:39,428 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=53.67 KB/54960 for 97ee24ee195dd7bff50b82d61598e8ac in 210ms, sequenceid=145, compaction requested=false 2024-12-08T03:58:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-08T03:58:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-08T03:58:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:39,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T03:58:39,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:39,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:39,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:39,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T03:58:39,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 836 msec 2024-12-08T03:58:39,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 847 msec 2024-12-08T03:58:39,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/6f49088ba90a4320926a41b7eea86d59 is 50, key is test_row_0/A:col10/1733630319431/Put/seqid=0 2024-12-08T03:58:39,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741864_1040 (size=14541) 2024-12-08T03:58:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630379472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630379473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733630319466, max timestamp=1733630319480Num uniques 859; 2024-12-08T03:58:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630379475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T03:58:39,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733630319466, max timestamp=1733630319480Num uniques 859; 2024-12-08T03:58:39,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T03:58:39,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630319468, max timestamp=1733630319480Num uniques 859; 2024-12-08T03:58:39,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:39,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630379541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630379546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630379587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630379587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630379598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,615 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/c4107be9646547bd8bd464a4f681a51c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c4107be9646547bd8bd464a4f681a51c 2024-12-08T03:58:39,627 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into c4107be9646547bd8bd464a4f681a51c(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:39,627 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:39,627 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=12, startTime=1733630319141; duration=0sec 2024-12-08T03:58:39,627 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:39,627 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630379658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630379658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T03:58:39,727 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:39,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-08T03:58:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:58:39,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:39,733 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:39,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:39,746 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/890c81f1afea447982453841cdbc3aa5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/890c81f1afea447982453841cdbc3aa5 2024-12-08T03:58:39,760 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 890c81f1afea447982453841cdbc3aa5(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:39,760 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:39,760 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=12, startTime=1733630319146; duration=0sec 2024-12-08T03:58:39,761 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:39,761 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630379798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630379798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630379808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:58:39,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/6f49088ba90a4320926a41b7eea86d59 2024-12-08T03:58:39,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/f6f1ef9c08d24e84948602014f689c6f is 50, key is test_row_0/B:col10/1733630319431/Put/seqid=0 2024-12-08T03:58:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630379867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630379870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:39,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741865_1041 (size=12151) 2024-12-08T03:58:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:39,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:39,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:58:39,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:39,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:39,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:58:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:58:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630380107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630380107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630380118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630380187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630380187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:58:40,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:40,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/f6f1ef9c08d24e84948602014f689c6f 2024-12-08T03:58:40,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/c4d89e2772424641ad4a2e98baa908c2 is 50, key is test_row_0/C:col10/1733630319431/Put/seqid=0 2024-12-08T03:58:40,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741866_1042 (size=12151) 2024-12-08T03:58:40,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/c4d89e2772424641ad4a2e98baa908c2 2024-12-08T03:58:40,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/6f49088ba90a4320926a41b7eea86d59 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6f49088ba90a4320926a41b7eea86d59 2024-12-08T03:58:40,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:58:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:58:40,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6f49088ba90a4320926a41b7eea86d59, entries=200, sequenceid=158, filesize=14.2 K 2024-12-08T03:58:40,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/f6f1ef9c08d24e84948602014f689c6f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f6f1ef9c08d24e84948602014f689c6f 2024-12-08T03:58:40,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f6f1ef9c08d24e84948602014f689c6f, entries=150, sequenceid=158, filesize=11.9 K 2024-12-08T03:58:40,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/c4d89e2772424641ad4a2e98baa908c2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/c4d89e2772424641ad4a2e98baa908c2 2024-12-08T03:58:40,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/c4d89e2772424641ad4a2e98baa908c2, entries=150, sequenceid=158, filesize=11.9 K 2024-12-08T03:58:40,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 97ee24ee195dd7bff50b82d61598e8ac in 960ms, sequenceid=158, compaction requested=true 2024-12-08T03:58:40,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:40,394 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:40,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:40,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:40,396 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:40,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:40,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:40,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:40,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:40,397 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:40,398 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:40,398 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,398 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c4107be9646547bd8bd464a4f681a51c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/562367154e164ae1ae7982b58d6c62e0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6f49088ba90a4320926a41b7eea86d59] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=38.0 K 2024-12-08T03:58:40,399 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4107be9646547bd8bd464a4f681a51c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733630317852 2024-12-08T03:58:40,399 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:40,399 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:40,400 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,400 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2250060066174d43b4f6001454ec070a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/70ac9ec62b5f483cb5b11248eb84910a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f6f1ef9c08d24e84948602014f689c6f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=35.7 K 2024-12-08T03:58:40,401 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562367154e164ae1ae7982b58d6c62e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733630318634 2024-12-08T03:58:40,402 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f49088ba90a4320926a41b7eea86d59, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733630319300 2024-12-08T03:58:40,403 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2250060066174d43b4f6001454ec070a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733630317852 2024-12-08T03:58:40,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ac9ec62b5f483cb5b11248eb84910a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733630318634 2024-12-08T03:58:40,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f1ef9c08d24e84948602014f689c6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733630319300 2024-12-08T03:58:40,422 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:40,423 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:40,423 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6c597390058c40989053a80073551a10 is 50, key is test_row_0/B:col10/1733630319431/Put/seqid=0 2024-12-08T03:58:40,424 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/e8221a0ed7874a179818cc10fcd7bfd1 is 50, key is test_row_0/A:col10/1733630319431/Put/seqid=0 2024-12-08T03:58:40,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741867_1043 (size=12493) 2024-12-08T03:58:40,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741868_1044 (size=12493) 2024-12-08T03:58:40,467 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/e8221a0ed7874a179818cc10fcd7bfd1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/e8221a0ed7874a179818cc10fcd7bfd1 2024-12-08T03:58:40,470 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6c597390058c40989053a80073551a10 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6c597390058c40989053a80073551a10 2024-12-08T03:58:40,480 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into e8221a0ed7874a179818cc10fcd7bfd1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:40,480 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:40,481 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630320394; duration=0sec 2024-12-08T03:58:40,481 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:40,481 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:40,481 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:40,483 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:40,483 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:40,483 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,483 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/890c81f1afea447982453841cdbc3aa5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/981e4bfd5290455d83866100d17d1360, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/c4d89e2772424641ad4a2e98baa908c2] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=35.7 K 2024-12-08T03:58:40,485 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890c81f1afea447982453841cdbc3aa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733630317852 2024-12-08T03:58:40,486 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981e4bfd5290455d83866100d17d1360, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733630318634 2024-12-08T03:58:40,486 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into 6c597390058c40989053a80073551a10(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:40,486 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:40,486 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630320396; duration=0sec 2024-12-08T03:58:40,486 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:40,486 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:40,486 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d89e2772424641ad4a2e98baa908c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733630319300 2024-12-08T03:58:40,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T03:58:40,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:40,503 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-12-08T03:58:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:40,510 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:40,510 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/15aae27b8de6461ca77435991744b787 is 50, key is test_row_0/C:col10/1733630319431/Put/seqid=0 2024-12-08T03:58:40,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3357b8f4600245bfa59b6b82753d8a75 is 50, key is test_row_0/A:col10/1733630319525/Put/seqid=0 2024-12-08T03:58:40,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741870_1046 (size=12151) 2024-12-08T03:58:40,534 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3357b8f4600245bfa59b6b82753d8a75 2024-12-08T03:58:40,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741869_1045 (size=12493) 2024-12-08T03:58:40,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/d48c8e2ae39445f9b4bcbdacd839ebc4 is 50, key is test_row_0/B:col10/1733630319525/Put/seqid=0 2024-12-08T03:58:40,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741871_1047 (size=12151) 2024-12-08T03:58:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:40,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630380636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630380638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630380640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630380697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630380697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630380748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630380748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630380748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:58:40,949 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/15aae27b8de6461ca77435991744b787 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/15aae27b8de6461ca77435991744b787 2024-12-08T03:58:40,962 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/d48c8e2ae39445f9b4bcbdacd839ebc4 2024-12-08T03:58:40,963 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 15aae27b8de6461ca77435991744b787(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:40,963 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:40,963 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630320396; duration=0sec 2024-12-08T03:58:40,963 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:40,963 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:40,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630380965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630380965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630380966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/658fc8fb3a1e4777bc809ba9966e71c6 is 50, key is test_row_0/C:col10/1733630319525/Put/seqid=0 2024-12-08T03:58:40,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741872_1048 (size=12151) 2024-12-08T03:58:40,982 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/658fc8fb3a1e4777bc809ba9966e71c6 2024-12-08T03:58:40,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3357b8f4600245bfa59b6b82753d8a75 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3357b8f4600245bfa59b6b82753d8a75 2024-12-08T03:58:40,999 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3357b8f4600245bfa59b6b82753d8a75, entries=150, sequenceid=195, filesize=11.9 K 2024-12-08T03:58:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/d48c8e2ae39445f9b4bcbdacd839ebc4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/d48c8e2ae39445f9b4bcbdacd839ebc4 2024-12-08T03:58:41,014 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/d48c8e2ae39445f9b4bcbdacd839ebc4, entries=150, sequenceid=195, filesize=11.9 K 2024-12-08T03:58:41,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/658fc8fb3a1e4777bc809ba9966e71c6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/658fc8fb3a1e4777bc809ba9966e71c6 2024-12-08T03:58:41,028 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/658fc8fb3a1e4777bc809ba9966e71c6, entries=150, sequenceid=195, filesize=11.9 K 2024-12-08T03:58:41,031 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 97ee24ee195dd7bff50b82d61598e8ac in 529ms, sequenceid=195, compaction requested=false 2024-12-08T03:58:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-08T03:58:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-08T03:58:41,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T03:58:41,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3070 sec 2024-12-08T03:58:41,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3190 sec 2024-12-08T03:58:41,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T03:58:41,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:41,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:41,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:41,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:41,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:41,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:41,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/db13a774178744a6a189dc4f76852f48 is 50, key is test_row_0/A:col10/1733630320634/Put/seqid=0 2024-12-08T03:58:41,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741873_1049 (size=12151) 2024-12-08T03:58:41,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630381316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630381316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630381317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630381427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630381427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630381428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630381637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630381637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630381638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/db13a774178744a6a189dc4f76852f48 2024-12-08T03:58:41,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630381715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630381718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,719 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 97ee24ee195dd7bff50b82d61598e8ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:58:41,719 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 97ee24ee195dd7bff50b82d61598e8ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:58:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/35f88977d8da4d28b9b5c96dc97034c3 is 50, key is test_row_0/B:col10/1733630320634/Put/seqid=0 2024-12-08T03:58:41,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741874_1050 (size=12151) 2024-12-08T03:58:41,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/35f88977d8da4d28b9b5c96dc97034c3 2024-12-08T03:58:41,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/81d36e312e604b76a6ea1e973f19109e is 50, key is test_row_0/C:col10/1733630320634/Put/seqid=0 2024-12-08T03:58:41,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741875_1051 (size=12151) 2024-12-08T03:58:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T03:58:41,877 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:41,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-08T03:58:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:58:41,881 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:41,882 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:41,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:41,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630381947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630381948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630381948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:58:42,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:58:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:42,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:42,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/81d36e312e604b76a6ea1e973f19109e 2024-12-08T03:58:42,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/db13a774178744a6a189dc4f76852f48 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/db13a774178744a6a189dc4f76852f48 2024-12-08T03:58:42,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/db13a774178744a6a189dc4f76852f48, entries=150, sequenceid=211, filesize=11.9 K 2024-12-08T03:58:42,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/35f88977d8da4d28b9b5c96dc97034c3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/35f88977d8da4d28b9b5c96dc97034c3 2024-12-08T03:58:42,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/35f88977d8da4d28b9b5c96dc97034c3, entries=150, sequenceid=211, filesize=11.9 K 2024-12-08T03:58:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/81d36e312e604b76a6ea1e973f19109e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/81d36e312e604b76a6ea1e973f19109e 2024-12-08T03:58:42,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:58:42,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:42,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:42,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:58:42,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/81d36e312e604b76a6ea1e973f19109e, entries=150, sequenceid=211, filesize=11.9 K 2024-12-08T03:58:42,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 97ee24ee195dd7bff50b82d61598e8ac in 918ms, sequenceid=211, compaction requested=true 2024-12-08T03:58:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:42,199 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:42,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:42,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T03:58:42,200 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:42,201 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:42,201 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:42,201 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,201 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/e8221a0ed7874a179818cc10fcd7bfd1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3357b8f4600245bfa59b6b82753d8a75, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/db13a774178744a6a189dc4f76852f48] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=35.9 K 2024-12-08T03:58:42,202 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:42,202 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:42,202 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,202 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6c597390058c40989053a80073551a10, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/d48c8e2ae39445f9b4bcbdacd839ebc4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/35f88977d8da4d28b9b5c96dc97034c3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=35.9 K 2024-12-08T03:58:42,202 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8221a0ed7874a179818cc10fcd7bfd1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733630319300 2024-12-08T03:58:42,203 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3357b8f4600245bfa59b6b82753d8a75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733630319525 2024-12-08T03:58:42,203 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c597390058c40989053a80073551a10, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733630319300 2024-12-08T03:58:42,204 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting db13a774178744a6a189dc4f76852f48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733630320634 2024-12-08T03:58:42,204 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d48c8e2ae39445f9b4bcbdacd839ebc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733630319525 2024-12-08T03:58:42,205 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f88977d8da4d28b9b5c96dc97034c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733630320634 2024-12-08T03:58:42,221 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:42,222 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/f2d3040f8170480eaa412e3b4a397cea is 50, key is test_row_0/A:col10/1733630320634/Put/seqid=0 2024-12-08T03:58:42,240 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:42,241 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/1080decffbc940498ecf8ea72453af68 is 50, key is test_row_0/B:col10/1733630320634/Put/seqid=0 2024-12-08T03:58:42,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741876_1052 (size=12595) 2024-12-08T03:58:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741877_1053 (size=12595) 2024-12-08T03:58:42,262 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/1080decffbc940498ecf8ea72453af68 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/1080decffbc940498ecf8ea72453af68 2024-12-08T03:58:42,271 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into 1080decffbc940498ecf8ea72453af68(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:42,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:42,272 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630322199; duration=0sec 2024-12-08T03:58:42,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:42,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:42,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:42,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:42,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:42,274 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,274 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/15aae27b8de6461ca77435991744b787, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/658fc8fb3a1e4777bc809ba9966e71c6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/81d36e312e604b76a6ea1e973f19109e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=35.9 K 2024-12-08T03:58:42,276 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 15aae27b8de6461ca77435991744b787, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733630319300 2024-12-08T03:58:42,277 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 658fc8fb3a1e4777bc809ba9966e71c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733630319525 2024-12-08T03:58:42,278 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d36e312e604b76a6ea1e973f19109e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733630320634 2024-12-08T03:58:42,291 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:42,292 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/f0da0bbb33764c5c86670b50efd83152 is 50, key is test_row_0/C:col10/1733630320634/Put/seqid=0 2024-12-08T03:58:42,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741878_1054 (size=12595) 2024-12-08T03:58:42,316 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/f0da0bbb33764c5c86670b50efd83152 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f0da0bbb33764c5c86670b50efd83152 2024-12-08T03:58:42,327 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into f0da0bbb33764c5c86670b50efd83152(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:42,328 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:42,328 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630322199; duration=0sec 2024-12-08T03:58:42,328 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:42,328 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:42,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T03:58:42,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,344 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T03:58:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/055b16d91a2f473380de27f24e49503d is 50, key is test_row_0/A:col10/1733630321310/Put/seqid=0 2024-12-08T03:58:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741879_1055 (size=12151) 2024-12-08T03:58:42,387 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/055b16d91a2f473380de27f24e49503d 2024-12-08T03:58:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/188f0c4dc00e4eefba674fc3f610f951 is 50, key is test_row_0/B:col10/1733630321310/Put/seqid=0 2024-12-08T03:58:42,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741880_1056 (size=12151) 2024-12-08T03:58:42,448 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/188f0c4dc00e4eefba674fc3f610f951 2024-12-08T03:58:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:42,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0e5cb25469d347d1abd67335d340f1cb is 50, key is test_row_0/C:col10/1733630321310/Put/seqid=0 2024-12-08T03:58:42,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741881_1057 (size=12151) 2024-12-08T03:58:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630382485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630382491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630382491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:58:42,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630382597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630382608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630382608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,667 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/f2d3040f8170480eaa412e3b4a397cea as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/f2d3040f8170480eaa412e3b4a397cea 2024-12-08T03:58:42,681 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into f2d3040f8170480eaa412e3b4a397cea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:42,681 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:42,681 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630322199; duration=0sec 2024-12-08T03:58:42,681 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:42,681 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:42,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630382807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630382818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630382818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:42,925 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0e5cb25469d347d1abd67335d340f1cb 2024-12-08T03:58:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/055b16d91a2f473380de27f24e49503d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/055b16d91a2f473380de27f24e49503d 2024-12-08T03:58:42,940 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/055b16d91a2f473380de27f24e49503d, entries=150, sequenceid=230, filesize=11.9 K 2024-12-08T03:58:42,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/188f0c4dc00e4eefba674fc3f610f951 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/188f0c4dc00e4eefba674fc3f610f951 2024-12-08T03:58:42,951 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/188f0c4dc00e4eefba674fc3f610f951, entries=150, sequenceid=230, filesize=11.9 K 2024-12-08T03:58:42,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0e5cb25469d347d1abd67335d340f1cb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0e5cb25469d347d1abd67335d340f1cb 2024-12-08T03:58:42,958 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0e5cb25469d347d1abd67335d340f1cb, entries=150, sequenceid=230, filesize=11.9 K 2024-12-08T03:58:42,960 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 97ee24ee195dd7bff50b82d61598e8ac in 615ms, sequenceid=230, compaction requested=false 2024-12-08T03:58:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-08T03:58:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-08T03:58:42,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-08T03:58:42,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-12-08T03:58:42,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0880 sec 2024-12-08T03:58:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T03:58:43,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:43,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-08T03:58:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:43,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:43,023 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:43,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:43,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T03:58:43,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:43,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:43,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:43,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:43,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:43,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:43,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/92b17b2164104a94b37c0c3396c1a0a2 is 50, key is test_row_0/A:col10/1733630323118/Put/seqid=0 2024-12-08T03:58:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741882_1058 (size=14541) 2024-12-08T03:58:43,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/92b17b2164104a94b37c0c3396c1a0a2 2024-12-08T03:58:43,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/8d065a034dc44a5b9238ba0830a589f7 is 50, key is test_row_0/B:col10/1733630323118/Put/seqid=0 2024-12-08T03:58:43,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630383146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630383146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741883_1059 (size=12151) 2024-12-08T03:58:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630383149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630383257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630383257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630383257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:43,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:43,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630383467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630383468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630383468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/8d065a034dc44a5b9238ba0830a589f7 2024-12-08T03:58:43,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/b8b277e7a3d24f2d986551c0341470c6 is 50, key is test_row_0/C:col10/1733630323118/Put/seqid=0 2024-12-08T03:58:43,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741884_1060 (size=12151) 2024-12-08T03:58:43,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630383727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,731 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 97ee24ee195dd7bff50b82d61598e8ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:58:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630383747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,749 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 97ee24ee195dd7bff50b82d61598e8ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:58:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630383777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630383777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630383787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:43,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:43,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:43,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:43,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/b8b277e7a3d24f2d986551c0341470c6 2024-12-08T03:58:43,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/92b17b2164104a94b37c0c3396c1a0a2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/92b17b2164104a94b37c0c3396c1a0a2 2024-12-08T03:58:43,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/92b17b2164104a94b37c0c3396c1a0a2, entries=200, sequenceid=248, filesize=14.2 K 2024-12-08T03:58:43,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/8d065a034dc44a5b9238ba0830a589f7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/8d065a034dc44a5b9238ba0830a589f7 2024-12-08T03:58:43,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/8d065a034dc44a5b9238ba0830a589f7, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T03:58:43,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/b8b277e7a3d24f2d986551c0341470c6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/b8b277e7a3d24f2d986551c0341470c6 2024-12-08T03:58:44,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/b8b277e7a3d24f2d986551c0341470c6, entries=150, sequenceid=248, filesize=11.9 K 2024-12-08T03:58:44,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 97ee24ee195dd7bff50b82d61598e8ac in 883ms, sequenceid=248, compaction requested=true 2024-12-08T03:58:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:44,003 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:44,004 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:44,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:44,004 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:44,004 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:44,005 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:44,005 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:44,005 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:44,005 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:44,005 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/f2d3040f8170480eaa412e3b4a397cea, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/055b16d91a2f473380de27f24e49503d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/92b17b2164104a94b37c0c3396c1a0a2] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=38.4 K 2024-12-08T03:58:44,005 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/1080decffbc940498ecf8ea72453af68, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/188f0c4dc00e4eefba674fc3f610f951, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/8d065a034dc44a5b9238ba0830a589f7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.0 K 2024-12-08T03:58:44,005 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2d3040f8170480eaa412e3b4a397cea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733630320634 2024-12-08T03:58:44,005 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1080decffbc940498ecf8ea72453af68, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733630320634 2024-12-08T03:58:44,006 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055b16d91a2f473380de27f24e49503d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733630321309 2024-12-08T03:58:44,006 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 188f0c4dc00e4eefba674fc3f610f951, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733630321309 2024-12-08T03:58:44,006 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b17b2164104a94b37c0c3396c1a0a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733630322478 2024-12-08T03:58:44,006 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d065a034dc44a5b9238ba0830a589f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733630322478 2024-12-08T03:58:44,020 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:44,020 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/fe5e2d16748d4efdaf9e03b5824ccb34 is 50, key is test_row_0/B:col10/1733630323118/Put/seqid=0 2024-12-08T03:58:44,021 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:44,022 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/321aa223d2134c84a40a26889dbced5c is 50, key is test_row_0/A:col10/1733630323118/Put/seqid=0 2024-12-08T03:58:44,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741885_1061 (size=12697) 2024-12-08T03:58:44,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741886_1062 (size=12697) 2024-12-08T03:58:44,055 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/321aa223d2134c84a40a26889dbced5c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/321aa223d2134c84a40a26889dbced5c 2024-12-08T03:58:44,064 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into 321aa223d2134c84a40a26889dbced5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:44,064 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:44,064 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630324003; duration=0sec 2024-12-08T03:58:44,064 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:44,064 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:44,064 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:44,068 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:44,068 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:44,068 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:44,068 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f0da0bbb33764c5c86670b50efd83152, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0e5cb25469d347d1abd67335d340f1cb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/b8b277e7a3d24f2d986551c0341470c6] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.0 K 2024-12-08T03:58:44,069 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0da0bbb33764c5c86670b50efd83152, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733630320634 2024-12-08T03:58:44,069 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5cb25469d347d1abd67335d340f1cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733630321309 2024-12-08T03:58:44,069 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b277e7a3d24f2d986551c0341470c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733630322478 2024-12-08T03:58:44,080 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:44,081 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/46f47981b6de471fa2b982db5a2b8f72 is 50, key is test_row_0/C:col10/1733630323118/Put/seqid=0 2024-12-08T03:58:44,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741887_1063 (size=12697) 2024-12-08T03:58:44,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T03:58:44,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:44,102 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T03:58:44,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:44,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:44,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/55ddedb435a140429d4dc7b7efa65893 is 50, key is test_row_0/A:col10/1733630323131/Put/seqid=0 2024-12-08T03:58:44,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741888_1064 (size=12251) 2024-12-08T03:58:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:44,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:44,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630384339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630384339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630384347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,439 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/fe5e2d16748d4efdaf9e03b5824ccb34 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/fe5e2d16748d4efdaf9e03b5824ccb34 2024-12-08T03:58:44,446 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into fe5e2d16748d4efdaf9e03b5824ccb34(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:44,446 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:44,446 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630324003; duration=0sec 2024-12-08T03:58:44,446 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:44,446 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630384458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630384458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630384458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,496 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/46f47981b6de471fa2b982db5a2b8f72 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/46f47981b6de471fa2b982db5a2b8f72 2024-12-08T03:58:44,505 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 46f47981b6de471fa2b982db5a2b8f72(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:44,505 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:44,505 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630324004; duration=0sec 2024-12-08T03:58:44,505 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:44,506 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:44,546 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/55ddedb435a140429d4dc7b7efa65893 2024-12-08T03:58:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/de6acd3de4ec408f95fc389cd907faef is 50, key is test_row_0/B:col10/1733630323131/Put/seqid=0 2024-12-08T03:58:44,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741889_1065 (size=12251) 2024-12-08T03:58:44,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630384667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630384667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630384667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,964 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/de6acd3de4ec408f95fc389cd907faef 2024-12-08T03:58:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/aadd8e2dda834c59b5bc84b9264942a9 is 50, key is test_row_0/C:col10/1733630323131/Put/seqid=0 2024-12-08T03:58:44,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630384977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630384977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630384979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:44,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741890_1066 (size=12251) 2024-12-08T03:58:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:45,398 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/aadd8e2dda834c59b5bc84b9264942a9 2024-12-08T03:58:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/55ddedb435a140429d4dc7b7efa65893 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/55ddedb435a140429d4dc7b7efa65893 2024-12-08T03:58:45,415 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/55ddedb435a140429d4dc7b7efa65893, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T03:58:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/de6acd3de4ec408f95fc389cd907faef as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/de6acd3de4ec408f95fc389cd907faef 2024-12-08T03:58:45,424 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/de6acd3de4ec408f95fc389cd907faef, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T03:58:45,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/aadd8e2dda834c59b5bc84b9264942a9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/aadd8e2dda834c59b5bc84b9264942a9 2024-12-08T03:58:45,434 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/aadd8e2dda834c59b5bc84b9264942a9, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T03:58:45,435 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 97ee24ee195dd7bff50b82d61598e8ac in 1333ms, sequenceid=264, compaction requested=false 2024-12-08T03:58:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:45,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-08T03:58:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-08T03:58:45,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T03:58:45,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4140 sec 2024-12-08T03:58:45,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4250 sec 2024-12-08T03:58:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:45,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T03:58:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:45,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:45,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/82036a6f0750464daefa3524649c2e96 is 50, key is test_row_0/A:col10/1733630324306/Put/seqid=0 2024-12-08T03:58:45,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741891_1067 (size=14741) 2024-12-08T03:58:45,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630385521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630385524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630385526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630385638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630385638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630385638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630385847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630385847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630385848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:45,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/82036a6f0750464daefa3524649c2e96 2024-12-08T03:58:45,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6a5d51f34aa244c38421bc6a83be0b05 is 50, key is test_row_0/B:col10/1733630324306/Put/seqid=0 2024-12-08T03:58:45,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741892_1068 (size=12301) 2024-12-08T03:58:45,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6a5d51f34aa244c38421bc6a83be0b05 2024-12-08T03:58:45,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/189c75aa812b48bfba04001c6c80da8d is 50, key is test_row_0/C:col10/1733630324306/Put/seqid=0 2024-12-08T03:58:45,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741893_1069 (size=12301) 2024-12-08T03:58:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630386157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630386157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630386157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/189c75aa812b48bfba04001c6c80da8d 2024-12-08T03:58:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/82036a6f0750464daefa3524649c2e96 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/82036a6f0750464daefa3524649c2e96 2024-12-08T03:58:46,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/82036a6f0750464daefa3524649c2e96, entries=200, sequenceid=284, filesize=14.4 K 2024-12-08T03:58:46,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6a5d51f34aa244c38421bc6a83be0b05 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6a5d51f34aa244c38421bc6a83be0b05 2024-12-08T03:58:46,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6a5d51f34aa244c38421bc6a83be0b05, entries=150, sequenceid=284, filesize=12.0 K 2024-12-08T03:58:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/189c75aa812b48bfba04001c6c80da8d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/189c75aa812b48bfba04001c6c80da8d 2024-12-08T03:58:46,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/189c75aa812b48bfba04001c6c80da8d, entries=150, sequenceid=284, filesize=12.0 K 2024-12-08T03:58:46,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 97ee24ee195dd7bff50b82d61598e8ac in 906ms, sequenceid=284, compaction requested=true 2024-12-08T03:58:46,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:46,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:46,399 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:46,399 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:46,402 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:46,402 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:46,402 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:46,402 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:46,402 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:46,402 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:46,402 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/fe5e2d16748d4efdaf9e03b5824ccb34, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/de6acd3de4ec408f95fc389cd907faef, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6a5d51f34aa244c38421bc6a83be0b05] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.4 K 2024-12-08T03:58:46,402 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/321aa223d2134c84a40a26889dbced5c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/55ddedb435a140429d4dc7b7efa65893, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/82036a6f0750464daefa3524649c2e96] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=38.8 K 2024-12-08T03:58:46,403 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting fe5e2d16748d4efdaf9e03b5824ccb34, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733630322478 2024-12-08T03:58:46,403 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321aa223d2134c84a40a26889dbced5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733630322478 2024-12-08T03:58:46,403 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting de6acd3de4ec408f95fc389cd907faef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733630323131 2024-12-08T03:58:46,404 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ddedb435a140429d4dc7b7efa65893, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733630323131 2024-12-08T03:58:46,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5d51f34aa244c38421bc6a83be0b05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733630324306 2024-12-08T03:58:46,404 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82036a6f0750464daefa3524649c2e96, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733630324306 2024-12-08T03:58:46,415 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:46,415 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:46,415 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/f37f27e9b4464088bb647ca261e2695d is 50, key is test_row_0/B:col10/1733630324306/Put/seqid=0 2024-12-08T03:58:46,415 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/af51904d33f440f99db9f41edc7ce8c3 is 50, key is test_row_0/A:col10/1733630324306/Put/seqid=0 2024-12-08T03:58:46,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741894_1070 (size=12949) 2024-12-08T03:58:46,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741895_1071 (size=12949) 2024-12-08T03:58:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:46,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T03:58:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:46,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/be8d43ffadee403fa95de743032871c5 is 50, key is test_row_0/A:col10/1733630326668/Put/seqid=0 2024-12-08T03:58:46,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741896_1072 (size=14741) 2024-12-08T03:58:46,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/be8d43ffadee403fa95de743032871c5 2024-12-08T03:58:46,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1 is 50, key is test_row_0/B:col10/1733630326668/Put/seqid=0 2024-12-08T03:58:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630386708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630386708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630386708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741897_1073 (size=12301) 2024-12-08T03:58:46,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1 2024-12-08T03:58:46,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6394e696238f4cd1a3333214fbbff675 is 50, key is test_row_0/C:col10/1733630326668/Put/seqid=0 2024-12-08T03:58:46,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741898_1074 (size=12301) 2024-12-08T03:58:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630386817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630386817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630386818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:46,841 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/f37f27e9b4464088bb647ca261e2695d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f37f27e9b4464088bb647ca261e2695d 2024-12-08T03:58:46,846 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/af51904d33f440f99db9f41edc7ce8c3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/af51904d33f440f99db9f41edc7ce8c3 2024-12-08T03:58:46,851 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into f37f27e9b4464088bb647ca261e2695d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:46,851 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:46,851 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630326399; duration=0sec 2024-12-08T03:58:46,851 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:46,851 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:46,852 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:46,854 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:46,854 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:46,854 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:46,854 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/46f47981b6de471fa2b982db5a2b8f72, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/aadd8e2dda834c59b5bc84b9264942a9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/189c75aa812b48bfba04001c6c80da8d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.4 K 2024-12-08T03:58:46,855 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f47981b6de471fa2b982db5a2b8f72, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733630322478 2024-12-08T03:58:46,855 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into af51904d33f440f99db9f41edc7ce8c3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:46,855 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:46,855 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630326398; duration=0sec 2024-12-08T03:58:46,855 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:46,855 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:46,856 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting aadd8e2dda834c59b5bc84b9264942a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733630323131 2024-12-08T03:58:46,857 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 189c75aa812b48bfba04001c6c80da8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733630324306 2024-12-08T03:58:46,888 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:46,888 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/707ef19909f04054bbb103498879865d is 50, key is test_row_0/C:col10/1733630324306/Put/seqid=0 2024-12-08T03:58:46,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741899_1075 (size=12949) 2024-12-08T03:58:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630387027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630387027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630387027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6394e696238f4cd1a3333214fbbff675 2024-12-08T03:58:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/be8d43ffadee403fa95de743032871c5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/be8d43ffadee403fa95de743032871c5 2024-12-08T03:58:47,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/be8d43ffadee403fa95de743032871c5, entries=200, sequenceid=300, filesize=14.4 K 2024-12-08T03:58:47,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1 2024-12-08T03:58:47,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1, entries=150, sequenceid=300, filesize=12.0 K 2024-12-08T03:58:47,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6394e696238f4cd1a3333214fbbff675 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6394e696238f4cd1a3333214fbbff675 2024-12-08T03:58:47,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6394e696238f4cd1a3333214fbbff675, entries=150, sequenceid=300, filesize=12.0 K 2024-12-08T03:58:47,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for 97ee24ee195dd7bff50b82d61598e8ac in 490ms, sequenceid=300, compaction requested=false 2024-12-08T03:58:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T03:58:47,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:47,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-08T03:58:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:58:47,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:47,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:47,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:58:47,311 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/707ef19909f04054bbb103498879865d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/707ef19909f04054bbb103498879865d 2024-12-08T03:58:47,317 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 707ef19909f04054bbb103498879865d(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:47,317 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:47,317 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630326399; duration=0sec 2024-12-08T03:58:47,317 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:47,317 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:47,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T03:58:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:47,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:58:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9494de927a534a9b8dd8bef82a8fdf7e is 50, key is test_row_0/A:col10/1733630326702/Put/seqid=0 2024-12-08T03:58:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741900_1076 (size=12301) 2024-12-08T03:58:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630387358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630387363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630387363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630387467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630387467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630387467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:58:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:58:47,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:58:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630387677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630387677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630387677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42232 deadline: 1733630387757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,759 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 97ee24ee195dd7bff50b82d61598e8ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:58:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9494de927a534a9b8dd8bef82a8fdf7e 2024-12-08T03:58:47,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/53e22e304cbc42dfbbdcca887424717e is 50, key is test_row_0/B:col10/1733630326702/Put/seqid=0 2024-12-08T03:58:47,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741901_1077 (size=12301) 2024-12-08T03:58:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42244 deadline: 1733630387778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:47,779 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 97ee24ee195dd7bff50b82d61598e8ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:58:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:47,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/53e22e304cbc42dfbbdcca887424717e 2024-12-08T03:58:47,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/2e6f5b7fed2844d5807169ec11bae83f is 50, key is test_row_0/C:col10/1733630326702/Put/seqid=0 2024-12-08T03:58:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741902_1078 (size=12301) 2024-12-08T03:58:47,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:58:47,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/2e6f5b7fed2844d5807169ec11bae83f 2024-12-08T03:58:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:58:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:47,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9494de927a534a9b8dd8bef82a8fdf7e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9494de927a534a9b8dd8bef82a8fdf7e 2024-12-08T03:58:47,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9494de927a534a9b8dd8bef82a8fdf7e, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T03:58:47,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/53e22e304cbc42dfbbdcca887424717e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/53e22e304cbc42dfbbdcca887424717e 2024-12-08T03:58:47,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/53e22e304cbc42dfbbdcca887424717e, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T03:58:47,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/2e6f5b7fed2844d5807169ec11bae83f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2e6f5b7fed2844d5807169ec11bae83f 2024-12-08T03:58:47,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2e6f5b7fed2844d5807169ec11bae83f, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T03:58:47,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 97ee24ee195dd7bff50b82d61598e8ac in 524ms, sequenceid=322, compaction requested=true 2024-12-08T03:58:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:47,862 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:47,863 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:47,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:47,864 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:47,864 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:47,864 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,864 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/af51904d33f440f99db9f41edc7ce8c3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/be8d43ffadee403fa95de743032871c5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9494de927a534a9b8dd8bef82a8fdf7e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=39.1 K 2024-12-08T03:58:47,864 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:47,864 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:47,865 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,865 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f37f27e9b4464088bb647ca261e2695d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/53e22e304cbc42dfbbdcca887424717e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.7 K 2024-12-08T03:58:47,865 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting af51904d33f440f99db9f41edc7ce8c3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733630324306 2024-12-08T03:58:47,865 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f37f27e9b4464088bb647ca261e2695d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733630324306 2024-12-08T03:58:47,865 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b138ec5dd2044a6a1fe0c3cd7d0a5d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733630325518 2024-12-08T03:58:47,865 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8d43ffadee403fa95de743032871c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733630325510 2024-12-08T03:58:47,866 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e22e304cbc42dfbbdcca887424717e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630326691 2024-12-08T03:58:47,866 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9494de927a534a9b8dd8bef82a8fdf7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630326691 2024-12-08T03:58:47,885 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:47,885 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3b76aba5204d4139b5999519b0a43cfb is 50, key is test_row_0/A:col10/1733630326702/Put/seqid=0 2024-12-08T03:58:47,904 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#67 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:47,905 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/e5ae4cb122f74d13af8f0ae7111bc93f is 50, key is test_row_0/B:col10/1733630326702/Put/seqid=0 2024-12-08T03:58:47,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741904_1080 (size=13051) 2024-12-08T03:58:47,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741903_1079 (size=13051) 2024-12-08T03:58:47,928 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/e5ae4cb122f74d13af8f0ae7111bc93f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/e5ae4cb122f74d13af8f0ae7111bc93f 2024-12-08T03:58:47,935 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into e5ae4cb122f74d13af8f0ae7111bc93f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:47,935 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:47,935 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630327863; duration=0sec 2024-12-08T03:58:47,936 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:47,936 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:47,936 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:47,937 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:47,937 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:47,937 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,937 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/707ef19909f04054bbb103498879865d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6394e696238f4cd1a3333214fbbff675, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2e6f5b7fed2844d5807169ec11bae83f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.7 K 2024-12-08T03:58:47,938 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 707ef19909f04054bbb103498879865d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733630324306 2024-12-08T03:58:47,938 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6394e696238f4cd1a3333214fbbff675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733630325518 2024-12-08T03:58:47,939 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6f5b7fed2844d5807169ec11bae83f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630326691 2024-12-08T03:58:47,957 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:47,957 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0c9dcd5f793f432fbf457774fef117c7 is 50, key is test_row_0/C:col10/1733630326702/Put/seqid=0 2024-12-08T03:58:47,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741905_1081 (size=13051) 2024-12-08T03:58:47,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T03:58:47,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:47,970 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T03:58:47,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:47,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:47,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:47,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:47,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:47,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/ed060380b0c6490fbee0e64bee8b8999 is 50, key is test_row_0/A:col10/1733630327354/Put/seqid=0 2024-12-08T03:58:47,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741906_1082 (size=12301) 2024-12-08T03:58:47,986 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/ed060380b0c6490fbee0e64bee8b8999 2024-12-08T03:58:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:47,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:47,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/3b4d6bfa0661403a9b1ad16d7e91d4d5 is 50, key is test_row_0/B:col10/1733630327354/Put/seqid=0 2024-12-08T03:58:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741907_1083 (size=12301) 2024-12-08T03:58:48,002 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/3b4d6bfa0661403a9b1ad16d7e91d4d5 2024-12-08T03:58:48,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/7fdf05594ae347ea8988342f1009b992 is 50, key is test_row_0/C:col10/1733630327354/Put/seqid=0 2024-12-08T03:58:48,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741908_1084 (size=12301) 2024-12-08T03:58:48,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630388022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630388022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630388022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630388127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630388127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630388137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:58:48,328 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3b76aba5204d4139b5999519b0a43cfb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3b76aba5204d4139b5999519b0a43cfb 2024-12-08T03:58:48,336 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into 3b76aba5204d4139b5999519b0a43cfb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:48,336 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:48,336 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630327862; duration=0sec 2024-12-08T03:58:48,336 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:48,336 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:48,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630388337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630388337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630388347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,369 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/0c9dcd5f793f432fbf457774fef117c7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0c9dcd5f793f432fbf457774fef117c7 2024-12-08T03:58:48,375 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 0c9dcd5f793f432fbf457774fef117c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:48,375 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:48,375 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630327863; duration=0sec 2024-12-08T03:58:48,375 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:48,375 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:48,414 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/7fdf05594ae347ea8988342f1009b992 2024-12-08T03:58:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/ed060380b0c6490fbee0e64bee8b8999 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ed060380b0c6490fbee0e64bee8b8999 2024-12-08T03:58:48,424 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ed060380b0c6490fbee0e64bee8b8999, entries=150, sequenceid=337, filesize=12.0 K 2024-12-08T03:58:48,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/3b4d6bfa0661403a9b1ad16d7e91d4d5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/3b4d6bfa0661403a9b1ad16d7e91d4d5 2024-12-08T03:58:48,432 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/3b4d6bfa0661403a9b1ad16d7e91d4d5, entries=150, sequenceid=337, filesize=12.0 K 2024-12-08T03:58:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/7fdf05594ae347ea8988342f1009b992 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7fdf05594ae347ea8988342f1009b992 2024-12-08T03:58:48,441 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7fdf05594ae347ea8988342f1009b992, entries=150, sequenceid=337, filesize=12.0 K 2024-12-08T03:58:48,442 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 97ee24ee195dd7bff50b82d61598e8ac in 472ms, sequenceid=337, compaction requested=false 2024-12-08T03:58:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-08T03:58:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-08T03:58:48,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T03:58:48,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2520 sec 2024-12-08T03:58:48,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2570 sec 2024-12-08T03:58:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:48,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-08T03:58:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:48,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3909b645f2c84fec9a7aacb8bd2a8475 is 50, key is test_row_0/A:col10/1733630328648/Put/seqid=0 2024-12-08T03:58:48,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741909_1085 (size=17181) 2024-12-08T03:58:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630388659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3909b645f2c84fec9a7aacb8bd2a8475 2024-12-08T03:58:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630388662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630388663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/c09d6814ac904f9b90a6467556c50a9b is 50, key is test_row_0/B:col10/1733630328648/Put/seqid=0 2024-12-08T03:58:48,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741910_1086 (size=12301) 2024-12-08T03:58:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630388767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630388768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630388777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630388978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630388978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:48,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630388987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/c09d6814ac904f9b90a6467556c50a9b 2024-12-08T03:58:49,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/28f883609a7d427db27e82e25744d8ff is 50, key is test_row_0/C:col10/1733630328648/Put/seqid=0 2024-12-08T03:58:49,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741911_1087 (size=12301) 2024-12-08T03:58:49,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630389287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630389287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630389297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T03:58:49,347 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:49,348 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:49,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-08T03:58:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:58:49,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:49,351 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:49,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:58:49,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/28f883609a7d427db27e82e25744d8ff 2024-12-08T03:58:49,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3909b645f2c84fec9a7aacb8bd2a8475 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3909b645f2c84fec9a7aacb8bd2a8475 2024-12-08T03:58:49,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T03:58:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:49,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:49,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:49,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3909b645f2c84fec9a7aacb8bd2a8475, entries=250, sequenceid=361, filesize=16.8 K 2024-12-08T03:58:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/c09d6814ac904f9b90a6467556c50a9b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/c09d6814ac904f9b90a6467556c50a9b 2024-12-08T03:58:49,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/c09d6814ac904f9b90a6467556c50a9b, entries=150, sequenceid=361, filesize=12.0 K 2024-12-08T03:58:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/28f883609a7d427db27e82e25744d8ff as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/28f883609a7d427db27e82e25744d8ff 2024-12-08T03:58:49,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/28f883609a7d427db27e82e25744d8ff, entries=150, sequenceid=361, filesize=12.0 K 2024-12-08T03:58:49,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 97ee24ee195dd7bff50b82d61598e8ac in 871ms, sequenceid=361, compaction requested=true 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:49,521 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:49,521 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:49,522 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:49,522 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:49,523 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:49,523 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:49,523 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:49,523 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:49,523 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/e5ae4cb122f74d13af8f0ae7111bc93f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/3b4d6bfa0661403a9b1ad16d7e91d4d5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/c09d6814ac904f9b90a6467556c50a9b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.8 K 2024-12-08T03:58:49,523 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3b76aba5204d4139b5999519b0a43cfb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ed060380b0c6490fbee0e64bee8b8999, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3909b645f2c84fec9a7aacb8bd2a8475] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=41.5 K 2024-12-08T03:58:49,523 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ae4cb122f74d13af8f0ae7111bc93f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630326691 2024-12-08T03:58:49,523 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b76aba5204d4139b5999519b0a43cfb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630326691 2024-12-08T03:58:49,524 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b4d6bfa0661403a9b1ad16d7e91d4d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733630327354 2024-12-08T03:58:49,524 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed060380b0c6490fbee0e64bee8b8999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733630327354 2024-12-08T03:58:49,524 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c09d6814ac904f9b90a6467556c50a9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630328008 2024-12-08T03:58:49,524 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3909b645f2c84fec9a7aacb8bd2a8475, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630328008 2024-12-08T03:58:49,532 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:49,532 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/2f7af7f7c43a4d288cca11b2d8247375 is 50, key is test_row_0/B:col10/1733630328648/Put/seqid=0 2024-12-08T03:58:49,532 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:49,533 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/383c737ad0e440fb8f9207de46caf3b3 is 50, key is test_row_0/A:col10/1733630328648/Put/seqid=0 2024-12-08T03:58:49,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741912_1088 (size=13153) 2024-12-08T03:58:49,560 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/2f7af7f7c43a4d288cca11b2d8247375 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2f7af7f7c43a4d288cca11b2d8247375 2024-12-08T03:58:49,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741913_1089 (size=13153) 2024-12-08T03:58:49,566 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into 2f7af7f7c43a4d288cca11b2d8247375(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:49,566 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:49,566 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630329521; duration=0sec 2024-12-08T03:58:49,567 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:49,567 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:49,567 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:49,569 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:49,569 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:49,569 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:49,569 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0c9dcd5f793f432fbf457774fef117c7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7fdf05594ae347ea8988342f1009b992, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/28f883609a7d427db27e82e25744d8ff] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.8 K 2024-12-08T03:58:49,570 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9dcd5f793f432fbf457774fef117c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630326691 2024-12-08T03:58:49,570 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fdf05594ae347ea8988342f1009b992, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733630327354 2024-12-08T03:58:49,571 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 28f883609a7d427db27e82e25744d8ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630328008 2024-12-08T03:58:49,580 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:49,581 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/888915a60d204295a5f50036d2dd4967 is 50, key is test_row_0/C:col10/1733630328648/Put/seqid=0 2024-12-08T03:58:49,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741914_1090 (size=13153) 2024-12-08T03:58:49,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T03:58:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:49,657 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:58:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:49,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9961af40e45d4b4aaa20e139c71b3dbc is 50, key is test_row_0/A:col10/1733630328660/Put/seqid=0 2024-12-08T03:58:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:58:49,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741915_1091 (size=12301) 2024-12-08T03:58:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:49,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630329805, max timestamp=1733630329826Num uniques 859; 2024-12-08T03:58:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630329805, max timestamp=1733630329826Num uniques 859; 2024-12-08T03:58:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630329805, max timestamp=1733630329826Num uniques 859; 2024-12-08T03:58:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:49,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:58:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630389850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630389854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630389857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,966 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/383c737ad0e440fb8f9207de46caf3b3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/383c737ad0e440fb8f9207de46caf3b3 2024-12-08T03:58:49,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630389967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630389967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630389968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:49,973 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into 383c737ad0e440fb8f9207de46caf3b3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:49,973 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:49,973 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630329521; duration=0sec 2024-12-08T03:58:49,973 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:49,973 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:58:49,991 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/888915a60d204295a5f50036d2dd4967 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/888915a60d204295a5f50036d2dd4967 2024-12-08T03:58:49,996 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 888915a60d204295a5f50036d2dd4967(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:49,996 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:49,996 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630329521; duration=0sec 2024-12-08T03:58:49,996 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:49,996 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:50,068 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9961af40e45d4b4aaa20e139c71b3dbc 2024-12-08T03:58:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/08f211eab87b42fabe79885a6cd9b479 is 50, key is test_row_0/B:col10/1733630328660/Put/seqid=0 2024-12-08T03:58:50,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741916_1092 (size=12301) 2024-12-08T03:58:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630390177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:50,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630390177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:50,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630390177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:50,485 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/08f211eab87b42fabe79885a6cd9b479 2024-12-08T03:58:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:58:50,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630390487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:50,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630390488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:50,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630390488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:50,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:50,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/5c438efe6ce5482eb1c1579ed7e8ddf7 is 50, key is test_row_0/C:col10/1733630328660/Put/seqid=0 2024-12-08T03:58:50,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741917_1093 (size=12301) 2024-12-08T03:58:50,907 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/5c438efe6ce5482eb1c1579ed7e8ddf7 2024-12-08T03:58:50,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9961af40e45d4b4aaa20e139c71b3dbc as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9961af40e45d4b4aaa20e139c71b3dbc 2024-12-08T03:58:50,918 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9961af40e45d4b4aaa20e139c71b3dbc, entries=150, sequenceid=372, filesize=12.0 K 2024-12-08T03:58:50,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/08f211eab87b42fabe79885a6cd9b479 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/08f211eab87b42fabe79885a6cd9b479 2024-12-08T03:58:50,925 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/08f211eab87b42fabe79885a6cd9b479, entries=150, sequenceid=372, filesize=12.0 K 2024-12-08T03:58:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/5c438efe6ce5482eb1c1579ed7e8ddf7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/5c438efe6ce5482eb1c1579ed7e8ddf7 2024-12-08T03:58:50,931 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/5c438efe6ce5482eb1c1579ed7e8ddf7, entries=150, sequenceid=372, filesize=12.0 K 2024-12-08T03:58:50,932 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 97ee24ee195dd7bff50b82d61598e8ac in 1275ms, sequenceid=372, compaction requested=false 2024-12-08T03:58:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:50,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-08T03:58:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-08T03:58:50,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-08T03:58:50,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-12-08T03:58:50,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5880 sec 2024-12-08T03:58:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:51,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-08T03:58:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T03:58:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T03:58:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T03:58:51,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/c03852e00f6744a7aceb6e9e4a62c443 is 50, key is test_row_0/A:col10/1733630329853/Put/seqid=0 2024-12-08T03:58:51,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630391008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630391009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630391011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741918_1094 (size=12301) 2024-12-08T03:58:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630391117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630391117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630391118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630391327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630391327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630391327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/c03852e00f6744a7aceb6e9e4a62c443 2024-12-08T03:58:51,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6749354d49c44abeabd15acef4d11406 is 50, key is test_row_0/B:col10/1733630329853/Put/seqid=0 2024-12-08T03:58:51,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741919_1095 (size=12301) 2024-12-08T03:58:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T03:58:51,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:51,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-08T03:58:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:58:51,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:51,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:51,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:58:51,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630391637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630391637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630391637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:51,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T03:58:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:51,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T03:58:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:51,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:58:51,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6749354d49c44abeabd15acef4d11406 2024-12-08T03:58:51,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/38c4b2359d2d4940a327d1b1f939ba88 is 50, key is test_row_0/C:col10/1733630329853/Put/seqid=0 2024-12-08T03:58:51,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741920_1096 (size=12301) 2024-12-08T03:58:51,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T03:58:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:51,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:52,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T03:58:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:52,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:52,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:58:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630392147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630392147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:52,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630392147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:52,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/38c4b2359d2d4940a327d1b1f939ba88 2024-12-08T03:58:52,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/c03852e00f6744a7aceb6e9e4a62c443 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c03852e00f6744a7aceb6e9e4a62c443 2024-12-08T03:58:52,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c03852e00f6744a7aceb6e9e4a62c443, entries=150, sequenceid=412, filesize=12.0 K 2024-12-08T03:58:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/6749354d49c44abeabd15acef4d11406 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6749354d49c44abeabd15acef4d11406 2024-12-08T03:58:52,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6749354d49c44abeabd15acef4d11406, entries=150, sequenceid=412, filesize=12.0 K 2024-12-08T03:58:52,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/38c4b2359d2d4940a327d1b1f939ba88 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/38c4b2359d2d4940a327d1b1f939ba88 2024-12-08T03:58:52,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/38c4b2359d2d4940a327d1b1f939ba88, entries=150, sequenceid=412, filesize=12.0 K 2024-12-08T03:58:52,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 97ee24ee195dd7bff50b82d61598e8ac in 1265ms, sequenceid=412, compaction requested=true 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:52,265 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:52,265 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:52,266 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:52,266 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:52,266 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:52,266 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:52,266 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:52,266 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:52,267 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2f7af7f7c43a4d288cca11b2d8247375, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/08f211eab87b42fabe79885a6cd9b479, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6749354d49c44abeabd15acef4d11406] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.9 K 2024-12-08T03:58:52,267 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/383c737ad0e440fb8f9207de46caf3b3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9961af40e45d4b4aaa20e139c71b3dbc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c03852e00f6744a7aceb6e9e4a62c443] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.9 K 2024-12-08T03:58:52,268 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7af7f7c43a4d288cca11b2d8247375, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630328008 2024-12-08T03:58:52,268 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 383c737ad0e440fb8f9207de46caf3b3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630328008 2024-12-08T03:58:52,269 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9961af40e45d4b4aaa20e139c71b3dbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733630328656 2024-12-08T03:58:52,269 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f211eab87b42fabe79885a6cd9b479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733630328656 2024-12-08T03:58:52,269 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6749354d49c44abeabd15acef4d11406, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733630329848 2024-12-08T03:58:52,269 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c03852e00f6744a7aceb6e9e4a62c443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733630329848 2024-12-08T03:58:52,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T03:58:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:52,275 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-08T03:58:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:52,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:52,280 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:52,280 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/14a59accc5384699b311eeea892fb4a4 is 50, key is test_row_0/A:col10/1733630329853/Put/seqid=0 2024-12-08T03:58:52,285 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c is 50, key is test_row_0/A:col10/1733630331005/Put/seqid=0 2024-12-08T03:58:52,285 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/bd1677307bbf4fe299886aa606ba13e0 is 50, key is test_row_0/B:col10/1733630329853/Put/seqid=0 2024-12-08T03:58:52,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741922_1098 (size=12301) 2024-12-08T03:58:52,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741923_1099 (size=13255) 2024-12-08T03:58:52,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741921_1097 (size=13255) 2024-12-08T03:58:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:58:52,711 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c 2024-12-08T03:58:52,728 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/bd1677307bbf4fe299886aa606ba13e0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/bd1677307bbf4fe299886aa606ba13e0 2024-12-08T03:58:52,728 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/14a59accc5384699b311eeea892fb4a4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/14a59accc5384699b311eeea892fb4a4 2024-12-08T03:58:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/99b832e932c6426784525c8ca52ddbf0 is 50, key is test_row_0/B:col10/1733630331005/Put/seqid=0 2024-12-08T03:58:52,736 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into bd1677307bbf4fe299886aa606ba13e0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:52,736 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into 14a59accc5384699b311eeea892fb4a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:52,736 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630332265; duration=0sec 2024-12-08T03:58:52,736 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630332265; duration=0sec 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:52,736 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:52,737 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:52,737 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:52,738 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:52,738 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/888915a60d204295a5f50036d2dd4967, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/5c438efe6ce5482eb1c1579ed7e8ddf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/38c4b2359d2d4940a327d1b1f939ba88] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=36.9 K 2024-12-08T03:58:52,738 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 888915a60d204295a5f50036d2dd4967, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630328008 2024-12-08T03:58:52,739 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c438efe6ce5482eb1c1579ed7e8ddf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733630328656 2024-12-08T03:58:52,740 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c4b2359d2d4940a327d1b1f939ba88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733630329848 2024-12-08T03:58:52,774 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:52,774 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/2c9a81ed6d1c4d2a9e7920754c960483 is 50, key is test_row_0/C:col10/1733630329853/Put/seqid=0 2024-12-08T03:58:52,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741924_1100 (size=12301) 2024-12-08T03:58:52,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741925_1101 (size=13255) 2024-12-08T03:58:52,807 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/2c9a81ed6d1c4d2a9e7920754c960483 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2c9a81ed6d1c4d2a9e7920754c960483 2024-12-08T03:58:52,814 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into 2c9a81ed6d1c4d2a9e7920754c960483(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:52,814 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:52,814 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630332265; duration=0sec 2024-12-08T03:58:52,815 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:52,815 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:53,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/99b832e932c6426784525c8ca52ddbf0 2024-12-08T03:58:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:53,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:53,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/e183398870ef479cbf22ab97005058cb is 50, key is test_row_0/C:col10/1733630331005/Put/seqid=0 2024-12-08T03:58:53,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741926_1102 (size=12301) 2024-12-08T03:58:53,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630333195, max timestamp=1733630333209Num uniques 859; 2024-12-08T03:58:53,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:58:53,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630333195, max timestamp=1733630333209Num uniques 859; 2024-12-08T03:58:53,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:58:53,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630333195, max timestamp=1733630333209Num uniques 859; 2024-12-08T03:58:53,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:58:53,214 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/e183398870ef479cbf22ab97005058cb 2024-12-08T03:58:53,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c 2024-12-08T03:58:53,226 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c, entries=150, sequenceid=421, filesize=12.0 K 2024-12-08T03:58:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/99b832e932c6426784525c8ca52ddbf0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/99b832e932c6426784525c8ca52ddbf0 2024-12-08T03:58:53,234 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/99b832e932c6426784525c8ca52ddbf0, entries=150, sequenceid=421, filesize=12.0 K 2024-12-08T03:58:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/e183398870ef479cbf22ab97005058cb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/e183398870ef479cbf22ab97005058cb 2024-12-08T03:58:53,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630393240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630393243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630393243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,249 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/e183398870ef479cbf22ab97005058cb, entries=150, sequenceid=421, filesize=12.0 K 2024-12-08T03:58:53,250 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 97ee24ee195dd7bff50b82d61598e8ac in 974ms, sequenceid=421, compaction requested=false 2024-12-08T03:58:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-08T03:58:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-08T03:58:53,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T03:58:53,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7430 sec 2024-12-08T03:58:53,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7490 sec 2024-12-08T03:58:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:53,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-08T03:58:53,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:53,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:53,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:53,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:53,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:53,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:58:53,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3149c598acf542a388bdf1461da2ee61 is 50, key is test_row_0/A:col10/1733630333238/Put/seqid=0 2024-12-08T03:58:53,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630393360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630393361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630393364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741927_1103 (size=14741) 2024-12-08T03:58:53,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630393467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630393477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630393477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T03:58:53,657 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:53,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:58:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-08T03:58:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:53,660 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:58:53,660 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:58:53,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:58:53,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630393677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630393687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630393687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:53,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3149c598acf542a388bdf1461da2ee61 2024-12-08T03:58:53,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/890fa170af724163aa1f5bfcee8478c1 is 50, key is test_row_0/B:col10/1733630333238/Put/seqid=0 2024-12-08T03:58:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741928_1104 (size=12301) 2024-12-08T03:58:53,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:53,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:53,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:53,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:53,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:53,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:53,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:53,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:53,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630393987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630394011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630394011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:54,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:54,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/890fa170af724163aa1f5bfcee8478c1 2024-12-08T03:58:54,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/1c1ad4dea93748708bf4cc265af87880 is 50, key is test_row_0/C:col10/1733630333238/Put/seqid=0 2024-12-08T03:58:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741929_1105 (size=12301) 2024-12-08T03:58:54,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:54,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42234 deadline: 1733630394497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42274 deadline: 1733630394517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:58:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42252 deadline: 1733630394517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:58:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:58:54,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:58:54,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/1c1ad4dea93748708bf4cc265af87880 2024-12-08T03:58:54,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/3149c598acf542a388bdf1461da2ee61 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3149c598acf542a388bdf1461da2ee61 2024-12-08T03:58:54,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3149c598acf542a388bdf1461da2ee61, entries=200, sequenceid=461, filesize=14.4 K 2024-12-08T03:58:54,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/890fa170af724163aa1f5bfcee8478c1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/890fa170af724163aa1f5bfcee8478c1 2024-12-08T03:58:54,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/890fa170af724163aa1f5bfcee8478c1, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T03:58:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/1c1ad4dea93748708bf4cc265af87880 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/1c1ad4dea93748708bf4cc265af87880 2024-12-08T03:58:54,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/1c1ad4dea93748708bf4cc265af87880, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T03:58:54,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 97ee24ee195dd7bff50b82d61598e8ac in 1282ms, sequenceid=461, compaction requested=true 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:54,630 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:54,630 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97ee24ee195dd7bff50b82d61598e8ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:58:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:54,631 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:54,631 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:54,631 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/B is initiating minor compaction (all files) 2024-12-08T03:58:54,631 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/A is initiating minor compaction (all files) 2024-12-08T03:58:54,632 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/A in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,632 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/B in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,632 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/bd1677307bbf4fe299886aa606ba13e0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/99b832e932c6426784525c8ca52ddbf0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/890fa170af724163aa1f5bfcee8478c1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=37.0 K 2024-12-08T03:58:54,632 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/14a59accc5384699b311eeea892fb4a4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3149c598acf542a388bdf1461da2ee61] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=39.4 K 2024-12-08T03:58:54,632 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a59accc5384699b311eeea892fb4a4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733630329848 2024-12-08T03:58:54,632 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting bd1677307bbf4fe299886aa606ba13e0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733630329848 2024-12-08T03:58:54,632 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae75e86b3eb44b6a8fdfc9fc5ab2aa9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733630331005 2024-12-08T03:58:54,632 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b832e932c6426784525c8ca52ddbf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733630331005 2024-12-08T03:58:54,633 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3149c598acf542a388bdf1461da2ee61, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733630333222 2024-12-08T03:58:54,633 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 890fa170af724163aa1f5bfcee8478c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733630333222 2024-12-08T03:58:54,644 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:54,644 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/dbf84c83b20d41068784a56cd8a59aae is 50, key is test_row_0/B:col10/1733630333238/Put/seqid=0 2024-12-08T03:58:54,646 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:54,646 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/23ea332466e042ffabccae807b7d600e is 50, key is test_row_0/A:col10/1733630333238/Put/seqid=0 2024-12-08T03:58:54,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741930_1106 (size=13357) 2024-12-08T03:58:54,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741931_1107 (size=13357) 2024-12-08T03:58:54,674 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/dbf84c83b20d41068784a56cd8a59aae as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/dbf84c83b20d41068784a56cd8a59aae 2024-12-08T03:58:54,676 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/23ea332466e042ffabccae807b7d600e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/23ea332466e042ffabccae807b7d600e 2024-12-08T03:58:54,683 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/B of 97ee24ee195dd7bff50b82d61598e8ac into dbf84c83b20d41068784a56cd8a59aae(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:54,683 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:54,683 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/B, priority=13, startTime=1733630334630; duration=0sec 2024-12-08T03:58:54,683 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:58:54,683 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:B 2024-12-08T03:58:54,684 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:58:54,685 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:58:54,685 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 97ee24ee195dd7bff50b82d61598e8ac/C is initiating minor compaction (all files) 2024-12-08T03:58:54,686 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97ee24ee195dd7bff50b82d61598e8ac/C in TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,686 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2c9a81ed6d1c4d2a9e7920754c960483, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/e183398870ef479cbf22ab97005058cb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/1c1ad4dea93748708bf4cc265af87880] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp, totalSize=37.0 K 2024-12-08T03:58:54,686 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9a81ed6d1c4d2a9e7920754c960483, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733630329848 2024-12-08T03:58:54,691 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e183398870ef479cbf22ab97005058cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1733630331005 2024-12-08T03:58:54,694 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c1ad4dea93748708bf4cc265af87880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733630333222 2024-12-08T03:58:54,694 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/A of 97ee24ee195dd7bff50b82d61598e8ac into 23ea332466e042ffabccae807b7d600e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:54,694 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:54,694 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/A, priority=13, startTime=1733630334630; duration=0sec 2024-12-08T03:58:54,694 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:54,694 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:A 2024-12-08T03:58:54,701 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97ee24ee195dd7bff50b82d61598e8ac#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:58:54,702 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/cd5e579d9888434696776140f903333e is 50, key is test_row_0/C:col10/1733630333238/Put/seqid=0 2024-12-08T03:58:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741932_1108 (size=13357) 2024-12-08T03:58:54,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T03:58:54,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:54,730 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T03:58:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/454bc680c2094c448e7e55bdbe7db24a is 50, key is test_row_0/A:col10/1733630333360/Put/seqid=0 2024-12-08T03:58:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741933_1109 (size=9857) 2024-12-08T03:58:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:55,122 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/cd5e579d9888434696776140f903333e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/cd5e579d9888434696776140f903333e 2024-12-08T03:58:55,128 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97ee24ee195dd7bff50b82d61598e8ac/C of 97ee24ee195dd7bff50b82d61598e8ac into cd5e579d9888434696776140f903333e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:58:55,128 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:55,128 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., storeName=97ee24ee195dd7bff50b82d61598e8ac/C, priority=13, startTime=1733630334630; duration=0sec 2024-12-08T03:58:55,128 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:58:55,128 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97ee24ee195dd7bff50b82d61598e8ac:C 2024-12-08T03:58:55,143 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/454bc680c2094c448e7e55bdbe7db24a 2024-12-08T03:58:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/0dcd0b2860734b199762de251445945f is 50, key is test_row_0/B:col10/1733630333360/Put/seqid=0 2024-12-08T03:58:55,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741934_1110 (size=9857) 2024-12-08T03:58:55,271 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-08T03:58:55,271 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-08T03:58:55,272 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,272 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,273 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-08T03:58:55,273 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,275 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-08T03:58:55,275 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,277 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,278 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,519 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-08T03:58:55,519 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,519 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,519 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:55,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. as already flushing 2024-12-08T03:58:55,528 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-08T03:58:55,528 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,528 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-08T03:58:55,528 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,528 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:55,528 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,528 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,528 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:55,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:55,561 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/0dcd0b2860734b199762de251445945f 2024-12-08T03:58:55,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6e7867bf4a8948ff8db20b297afa56e4 is 50, key is test_row_0/C:col10/1733630333360/Put/seqid=0 2024-12-08T03:58:55,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741935_1111 (size=9857) 2024-12-08T03:58:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:55,983 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6e7867bf4a8948ff8db20b297afa56e4 2024-12-08T03:58:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/454bc680c2094c448e7e55bdbe7db24a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/454bc680c2094c448e7e55bdbe7db24a 2024-12-08T03:58:56,002 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/454bc680c2094c448e7e55bdbe7db24a, entries=100, sequenceid=471, filesize=9.6 K 2024-12-08T03:58:56,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/0dcd0b2860734b199762de251445945f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0dcd0b2860734b199762de251445945f 2024-12-08T03:58:56,008 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0dcd0b2860734b199762de251445945f, entries=100, sequenceid=471, filesize=9.6 K 2024-12-08T03:58:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/6e7867bf4a8948ff8db20b297afa56e4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6e7867bf4a8948ff8db20b297afa56e4 2024-12-08T03:58:56,013 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6e7867bf4a8948ff8db20b297afa56e4, entries=100, sequenceid=471, filesize=9.6 K 2024-12-08T03:58:56,014 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=20.13 KB/20610 for 97ee24ee195dd7bff50b82d61598e8ac in 1284ms, sequenceid=471, compaction requested=false 2024-12-08T03:58:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 97ee24ee195dd7bff50b82d61598e8ac: 2024-12-08T03:58:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-08T03:58:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-08T03:58:56,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T03:58:56,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-12-08T03:58:56,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3590 sec 2024-12-08T03:58:56,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:58:57,833 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-08T03:58:57,833 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:57,833 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:57,833 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:57,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T03:58:57,838 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:58:57,849 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-08T03:58:57,850 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:58:57,850 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:57,850 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:57,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:57,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:58:57,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-08T03:58:57,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4366 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4275 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1881 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5640 rows 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1865 2024-12-08T03:58:57,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5595 rows 2024-12-08T03:58:57,851 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:58:57,852 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:58:57,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:57,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:58:57,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:58:57,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:58:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:58:57,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630337865"}]},"ts":"1733630337865"} 2024-12-08T03:58:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:58:57,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:58:57,866 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:58:57,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:58:57,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, UNASSIGN}] 2024-12-08T03:58:57,873 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, UNASSIGN 2024-12-08T03:58:57,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=97ee24ee195dd7bff50b82d61598e8ac, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:58:57,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, UNASSIGN because future has completed 2024-12-08T03:58:57,878 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:58:57,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:58:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:58:58,035 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:58,035 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:58:58,036 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 97ee24ee195dd7bff50b82d61598e8ac, disabling compactions & flushes 2024-12-08T03:58:58,036 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:58,036 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:58,036 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. after waiting 0 ms 2024-12-08T03:58:58,036 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:58,036 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 97ee24ee195dd7bff50b82d61598e8ac 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T03:58:58,037 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=A 2024-12-08T03:58:58,037 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:58,037 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=B 2024-12-08T03:58:58,037 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:58,037 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97ee24ee195dd7bff50b82d61598e8ac, store=C 2024-12-08T03:58:58,037 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:58:58,044 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9e67e1fdbd034ef5aaa7ba859157adf5 is 50, key is test_row_0/A:col10/1733630335527/Put/seqid=0 2024-12-08T03:58:58,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741936_1112 (size=12301) 2024-12-08T03:58:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:58:58,452 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9e67e1fdbd034ef5aaa7ba859157adf5 2024-12-08T03:58:58,484 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/637a1ccf4cbe4968a8e2c4d02af60f6a is 50, key is test_row_0/B:col10/1733630335527/Put/seqid=0 2024-12-08T03:58:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:58:58,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741937_1113 (size=12301) 2024-12-08T03:58:58,891 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/637a1ccf4cbe4968a8e2c4d02af60f6a 2024-12-08T03:58:58,904 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/7319c2d15d334bc9ac38fc264f43a658 is 50, key is test_row_0/C:col10/1733630335527/Put/seqid=0 2024-12-08T03:58:58,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741938_1114 (size=12301) 2024-12-08T03:58:58,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:58:59,310 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/7319c2d15d334bc9ac38fc264f43a658 2024-12-08T03:58:59,320 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/A/9e67e1fdbd034ef5aaa7ba859157adf5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9e67e1fdbd034ef5aaa7ba859157adf5 2024-12-08T03:58:59,325 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9e67e1fdbd034ef5aaa7ba859157adf5, entries=150, sequenceid=480, filesize=12.0 K 2024-12-08T03:58:59,326 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/B/637a1ccf4cbe4968a8e2c4d02af60f6a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/637a1ccf4cbe4968a8e2c4d02af60f6a 2024-12-08T03:58:59,332 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/637a1ccf4cbe4968a8e2c4d02af60f6a, entries=150, sequenceid=480, filesize=12.0 K 2024-12-08T03:58:59,333 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/.tmp/C/7319c2d15d334bc9ac38fc264f43a658 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7319c2d15d334bc9ac38fc264f43a658 2024-12-08T03:58:59,338 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7319c2d15d334bc9ac38fc264f43a658, entries=150, sequenceid=480, filesize=12.0 K 2024-12-08T03:58:59,339 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 97ee24ee195dd7bff50b82d61598e8ac in 1303ms, sequenceid=480, compaction requested=true 2024-12-08T03:58:59,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/bf36a2ada2b9487d9be511bc95a60252, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/2fd39d6bfbbd46bc9077ca288ca70a5e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6fcc2d58eb924ea6a6a65d0fad42c01e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4fbacb4393ce4404b0404bc13a87db18, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4beb5da98d464c0daf569ce6becafbe5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/835d4f2caf8b4af4965f5db13a56b496, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c4107be9646547bd8bd464a4f681a51c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/df99fd0072344182bdd28dd809b2cc9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/562367154e164ae1ae7982b58d6c62e0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6f49088ba90a4320926a41b7eea86d59, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/e8221a0ed7874a179818cc10fcd7bfd1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3357b8f4600245bfa59b6b82753d8a75, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/f2d3040f8170480eaa412e3b4a397cea, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/db13a774178744a6a189dc4f76852f48, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/055b16d91a2f473380de27f24e49503d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/92b17b2164104a94b37c0c3396c1a0a2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/321aa223d2134c84a40a26889dbced5c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/55ddedb435a140429d4dc7b7efa65893, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/82036a6f0750464daefa3524649c2e96, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/af51904d33f440f99db9f41edc7ce8c3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/be8d43ffadee403fa95de743032871c5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3b76aba5204d4139b5999519b0a43cfb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9494de927a534a9b8dd8bef82a8fdf7e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ed060380b0c6490fbee0e64bee8b8999, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3909b645f2c84fec9a7aacb8bd2a8475, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/383c737ad0e440fb8f9207de46caf3b3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9961af40e45d4b4aaa20e139c71b3dbc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/14a59accc5384699b311eeea892fb4a4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c03852e00f6744a7aceb6e9e4a62c443, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3149c598acf542a388bdf1461da2ee61] to archive 2024-12-08T03:58:59,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:58:59,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/bf36a2ada2b9487d9be511bc95a60252 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/bf36a2ada2b9487d9be511bc95a60252 2024-12-08T03:58:59,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/2fd39d6bfbbd46bc9077ca288ca70a5e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/2fd39d6bfbbd46bc9077ca288ca70a5e 2024-12-08T03:58:59,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6fcc2d58eb924ea6a6a65d0fad42c01e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6fcc2d58eb924ea6a6a65d0fad42c01e 2024-12-08T03:58:59,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4fbacb4393ce4404b0404bc13a87db18 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4fbacb4393ce4404b0404bc13a87db18 2024-12-08T03:58:59,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4beb5da98d464c0daf569ce6becafbe5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/4beb5da98d464c0daf569ce6becafbe5 2024-12-08T03:58:59,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/835d4f2caf8b4af4965f5db13a56b496 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/835d4f2caf8b4af4965f5db13a56b496 2024-12-08T03:58:59,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c4107be9646547bd8bd464a4f681a51c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c4107be9646547bd8bd464a4f681a51c 2024-12-08T03:58:59,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/df99fd0072344182bdd28dd809b2cc9c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/df99fd0072344182bdd28dd809b2cc9c 2024-12-08T03:58:59,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/562367154e164ae1ae7982b58d6c62e0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/562367154e164ae1ae7982b58d6c62e0 2024-12-08T03:58:59,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6f49088ba90a4320926a41b7eea86d59 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/6f49088ba90a4320926a41b7eea86d59 2024-12-08T03:58:59,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/e8221a0ed7874a179818cc10fcd7bfd1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/e8221a0ed7874a179818cc10fcd7bfd1 2024-12-08T03:58:59,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3357b8f4600245bfa59b6b82753d8a75 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3357b8f4600245bfa59b6b82753d8a75 2024-12-08T03:58:59,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/f2d3040f8170480eaa412e3b4a397cea to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/f2d3040f8170480eaa412e3b4a397cea 2024-12-08T03:58:59,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/db13a774178744a6a189dc4f76852f48 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/db13a774178744a6a189dc4f76852f48 2024-12-08T03:58:59,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/055b16d91a2f473380de27f24e49503d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/055b16d91a2f473380de27f24e49503d 2024-12-08T03:58:59,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/92b17b2164104a94b37c0c3396c1a0a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/92b17b2164104a94b37c0c3396c1a0a2 2024-12-08T03:58:59,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/321aa223d2134c84a40a26889dbced5c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/321aa223d2134c84a40a26889dbced5c 2024-12-08T03:58:59,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/55ddedb435a140429d4dc7b7efa65893 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/55ddedb435a140429d4dc7b7efa65893 2024-12-08T03:58:59,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/82036a6f0750464daefa3524649c2e96 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/82036a6f0750464daefa3524649c2e96 2024-12-08T03:58:59,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/af51904d33f440f99db9f41edc7ce8c3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/af51904d33f440f99db9f41edc7ce8c3 2024-12-08T03:58:59,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/be8d43ffadee403fa95de743032871c5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/be8d43ffadee403fa95de743032871c5 2024-12-08T03:58:59,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3b76aba5204d4139b5999519b0a43cfb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3b76aba5204d4139b5999519b0a43cfb 2024-12-08T03:58:59,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9494de927a534a9b8dd8bef82a8fdf7e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9494de927a534a9b8dd8bef82a8fdf7e 2024-12-08T03:58:59,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ed060380b0c6490fbee0e64bee8b8999 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ed060380b0c6490fbee0e64bee8b8999 2024-12-08T03:58:59,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3909b645f2c84fec9a7aacb8bd2a8475 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3909b645f2c84fec9a7aacb8bd2a8475 2024-12-08T03:58:59,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/383c737ad0e440fb8f9207de46caf3b3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/383c737ad0e440fb8f9207de46caf3b3 2024-12-08T03:58:59,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9961af40e45d4b4aaa20e139c71b3dbc to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9961af40e45d4b4aaa20e139c71b3dbc 2024-12-08T03:58:59,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/14a59accc5384699b311eeea892fb4a4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/14a59accc5384699b311eeea892fb4a4 2024-12-08T03:58:59,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c03852e00f6744a7aceb6e9e4a62c443 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/c03852e00f6744a7aceb6e9e4a62c443 2024-12-08T03:58:59,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/ae75e86b3eb44b6a8fdfc9fc5ab2aa9c 2024-12-08T03:58:59,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3149c598acf542a388bdf1461da2ee61 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/3149c598acf542a388bdf1461da2ee61 2024-12-08T03:58:59,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/b63f746f735143d9aff527e13670d16b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/32758d1e325846b4a7d5ee36663ea904, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/50a35c1eea8d4ee990c939a20268e2a8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/11b9d6cb63724d3ea55369184cc5cb66, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6d6e137826014dc19c50fa86f0de4c88, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/242aa16b88c043e3b3d7a50df1b27619, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2250060066174d43b4f6001454ec070a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/ebeec46810474c8f9bc5611f665d3b30, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/70ac9ec62b5f483cb5b11248eb84910a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6c597390058c40989053a80073551a10, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f6f1ef9c08d24e84948602014f689c6f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/d48c8e2ae39445f9b4bcbdacd839ebc4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/1080decffbc940498ecf8ea72453af68, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/35f88977d8da4d28b9b5c96dc97034c3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/188f0c4dc00e4eefba674fc3f610f951, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/fe5e2d16748d4efdaf9e03b5824ccb34, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/8d065a034dc44a5b9238ba0830a589f7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/de6acd3de4ec408f95fc389cd907faef, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f37f27e9b4464088bb647ca261e2695d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6a5d51f34aa244c38421bc6a83be0b05, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/e5ae4cb122f74d13af8f0ae7111bc93f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/53e22e304cbc42dfbbdcca887424717e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/3b4d6bfa0661403a9b1ad16d7e91d4d5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2f7af7f7c43a4d288cca11b2d8247375, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/c09d6814ac904f9b90a6467556c50a9b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/08f211eab87b42fabe79885a6cd9b479, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/bd1677307bbf4fe299886aa606ba13e0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6749354d49c44abeabd15acef4d11406, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/99b832e932c6426784525c8ca52ddbf0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/890fa170af724163aa1f5bfcee8478c1] to archive 2024-12-08T03:58:59,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:58:59,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/b63f746f735143d9aff527e13670d16b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/b63f746f735143d9aff527e13670d16b 2024-12-08T03:58:59,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/32758d1e325846b4a7d5ee36663ea904 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/32758d1e325846b4a7d5ee36663ea904 2024-12-08T03:58:59,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/50a35c1eea8d4ee990c939a20268e2a8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/50a35c1eea8d4ee990c939a20268e2a8 2024-12-08T03:58:59,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/11b9d6cb63724d3ea55369184cc5cb66 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/11b9d6cb63724d3ea55369184cc5cb66 2024-12-08T03:58:59,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6d6e137826014dc19c50fa86f0de4c88 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6d6e137826014dc19c50fa86f0de4c88 2024-12-08T03:58:59,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/242aa16b88c043e3b3d7a50df1b27619 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/242aa16b88c043e3b3d7a50df1b27619 2024-12-08T03:58:59,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2250060066174d43b4f6001454ec070a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2250060066174d43b4f6001454ec070a 2024-12-08T03:58:59,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/ebeec46810474c8f9bc5611f665d3b30 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/ebeec46810474c8f9bc5611f665d3b30 2024-12-08T03:58:59,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/70ac9ec62b5f483cb5b11248eb84910a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/70ac9ec62b5f483cb5b11248eb84910a 2024-12-08T03:58:59,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6c597390058c40989053a80073551a10 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6c597390058c40989053a80073551a10 2024-12-08T03:58:59,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f6f1ef9c08d24e84948602014f689c6f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f6f1ef9c08d24e84948602014f689c6f 2024-12-08T03:58:59,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/d48c8e2ae39445f9b4bcbdacd839ebc4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/d48c8e2ae39445f9b4bcbdacd839ebc4 2024-12-08T03:58:59,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/1080decffbc940498ecf8ea72453af68 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/1080decffbc940498ecf8ea72453af68 2024-12-08T03:58:59,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/35f88977d8da4d28b9b5c96dc97034c3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/35f88977d8da4d28b9b5c96dc97034c3 2024-12-08T03:58:59,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/188f0c4dc00e4eefba674fc3f610f951 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/188f0c4dc00e4eefba674fc3f610f951 2024-12-08T03:58:59,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/fe5e2d16748d4efdaf9e03b5824ccb34 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/fe5e2d16748d4efdaf9e03b5824ccb34 2024-12-08T03:58:59,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/8d065a034dc44a5b9238ba0830a589f7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/8d065a034dc44a5b9238ba0830a589f7 2024-12-08T03:58:59,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/de6acd3de4ec408f95fc389cd907faef to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/de6acd3de4ec408f95fc389cd907faef 2024-12-08T03:58:59,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f37f27e9b4464088bb647ca261e2695d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/f37f27e9b4464088bb647ca261e2695d 2024-12-08T03:58:59,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6a5d51f34aa244c38421bc6a83be0b05 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6a5d51f34aa244c38421bc6a83be0b05 2024-12-08T03:58:59,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0b138ec5dd2044a6a1fe0c3cd7d0a5d1 2024-12-08T03:58:59,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/e5ae4cb122f74d13af8f0ae7111bc93f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/e5ae4cb122f74d13af8f0ae7111bc93f 2024-12-08T03:58:59,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/53e22e304cbc42dfbbdcca887424717e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/53e22e304cbc42dfbbdcca887424717e 2024-12-08T03:58:59,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/3b4d6bfa0661403a9b1ad16d7e91d4d5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/3b4d6bfa0661403a9b1ad16d7e91d4d5 2024-12-08T03:58:59,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2f7af7f7c43a4d288cca11b2d8247375 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/2f7af7f7c43a4d288cca11b2d8247375 2024-12-08T03:58:59,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/c09d6814ac904f9b90a6467556c50a9b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/c09d6814ac904f9b90a6467556c50a9b 2024-12-08T03:58:59,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/08f211eab87b42fabe79885a6cd9b479 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/08f211eab87b42fabe79885a6cd9b479 2024-12-08T03:58:59,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/bd1677307bbf4fe299886aa606ba13e0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/bd1677307bbf4fe299886aa606ba13e0 2024-12-08T03:58:59,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6749354d49c44abeabd15acef4d11406 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/6749354d49c44abeabd15acef4d11406 2024-12-08T03:58:59,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/99b832e932c6426784525c8ca52ddbf0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/99b832e932c6426784525c8ca52ddbf0 2024-12-08T03:58:59,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/890fa170af724163aa1f5bfcee8478c1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/890fa170af724163aa1f5bfcee8478c1 2024-12-08T03:58:59,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/bc0803c5eedd49058c0fb1a3eec9765b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/253dc4e95322439ea219093718d6195d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6c33171b7e1d4ee2a6ebcd333951b33f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f7e106432b2841d6910e6629d416d437, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0bb7639f856e45678fcfc1e14f586fff, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/41c8cff045d24bb7bacf361ac843e36e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/890c81f1afea447982453841cdbc3aa5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/4ec9a0fc7a9548b9b6502f1bef72e9c0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/981e4bfd5290455d83866100d17d1360, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/15aae27b8de6461ca77435991744b787, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/c4d89e2772424641ad4a2e98baa908c2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/658fc8fb3a1e4777bc809ba9966e71c6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f0da0bbb33764c5c86670b50efd83152, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/81d36e312e604b76a6ea1e973f19109e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0e5cb25469d347d1abd67335d340f1cb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/46f47981b6de471fa2b982db5a2b8f72, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/b8b277e7a3d24f2d986551c0341470c6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/aadd8e2dda834c59b5bc84b9264942a9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/707ef19909f04054bbb103498879865d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/189c75aa812b48bfba04001c6c80da8d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6394e696238f4cd1a3333214fbbff675, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0c9dcd5f793f432fbf457774fef117c7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2e6f5b7fed2844d5807169ec11bae83f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7fdf05594ae347ea8988342f1009b992, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/888915a60d204295a5f50036d2dd4967, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/28f883609a7d427db27e82e25744d8ff, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/5c438efe6ce5482eb1c1579ed7e8ddf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2c9a81ed6d1c4d2a9e7920754c960483, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/38c4b2359d2d4940a327d1b1f939ba88, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/e183398870ef479cbf22ab97005058cb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/1c1ad4dea93748708bf4cc265af87880] to archive 2024-12-08T03:58:59,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:58:59,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/bc0803c5eedd49058c0fb1a3eec9765b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/bc0803c5eedd49058c0fb1a3eec9765b 2024-12-08T03:58:59,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/253dc4e95322439ea219093718d6195d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/253dc4e95322439ea219093718d6195d 2024-12-08T03:58:59,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6c33171b7e1d4ee2a6ebcd333951b33f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6c33171b7e1d4ee2a6ebcd333951b33f 2024-12-08T03:58:59,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f7e106432b2841d6910e6629d416d437 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f7e106432b2841d6910e6629d416d437 2024-12-08T03:58:59,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0bb7639f856e45678fcfc1e14f586fff to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0bb7639f856e45678fcfc1e14f586fff 2024-12-08T03:58:59,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/41c8cff045d24bb7bacf361ac843e36e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/41c8cff045d24bb7bacf361ac843e36e 2024-12-08T03:58:59,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/890c81f1afea447982453841cdbc3aa5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/890c81f1afea447982453841cdbc3aa5 2024-12-08T03:58:59,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/4ec9a0fc7a9548b9b6502f1bef72e9c0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/4ec9a0fc7a9548b9b6502f1bef72e9c0 2024-12-08T03:58:59,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/981e4bfd5290455d83866100d17d1360 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/981e4bfd5290455d83866100d17d1360 2024-12-08T03:58:59,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/15aae27b8de6461ca77435991744b787 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/15aae27b8de6461ca77435991744b787 2024-12-08T03:58:59,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/c4d89e2772424641ad4a2e98baa908c2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/c4d89e2772424641ad4a2e98baa908c2 2024-12-08T03:58:59,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/658fc8fb3a1e4777bc809ba9966e71c6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/658fc8fb3a1e4777bc809ba9966e71c6 2024-12-08T03:58:59,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f0da0bbb33764c5c86670b50efd83152 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/f0da0bbb33764c5c86670b50efd83152 2024-12-08T03:58:59,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/81d36e312e604b76a6ea1e973f19109e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/81d36e312e604b76a6ea1e973f19109e 2024-12-08T03:58:59,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0e5cb25469d347d1abd67335d340f1cb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0e5cb25469d347d1abd67335d340f1cb 2024-12-08T03:58:59,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/46f47981b6de471fa2b982db5a2b8f72 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/46f47981b6de471fa2b982db5a2b8f72 2024-12-08T03:58:59,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/b8b277e7a3d24f2d986551c0341470c6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/b8b277e7a3d24f2d986551c0341470c6 2024-12-08T03:58:59,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/aadd8e2dda834c59b5bc84b9264942a9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/aadd8e2dda834c59b5bc84b9264942a9 2024-12-08T03:58:59,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/707ef19909f04054bbb103498879865d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/707ef19909f04054bbb103498879865d 2024-12-08T03:58:59,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/189c75aa812b48bfba04001c6c80da8d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/189c75aa812b48bfba04001c6c80da8d 2024-12-08T03:58:59,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6394e696238f4cd1a3333214fbbff675 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6394e696238f4cd1a3333214fbbff675 2024-12-08T03:58:59,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0c9dcd5f793f432fbf457774fef117c7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/0c9dcd5f793f432fbf457774fef117c7 2024-12-08T03:58:59,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2e6f5b7fed2844d5807169ec11bae83f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2e6f5b7fed2844d5807169ec11bae83f 2024-12-08T03:58:59,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7fdf05594ae347ea8988342f1009b992 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7fdf05594ae347ea8988342f1009b992 2024-12-08T03:58:59,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/888915a60d204295a5f50036d2dd4967 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/888915a60d204295a5f50036d2dd4967 2024-12-08T03:58:59,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/28f883609a7d427db27e82e25744d8ff to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/28f883609a7d427db27e82e25744d8ff 2024-12-08T03:58:59,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/5c438efe6ce5482eb1c1579ed7e8ddf7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/5c438efe6ce5482eb1c1579ed7e8ddf7 2024-12-08T03:58:59,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2c9a81ed6d1c4d2a9e7920754c960483 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/2c9a81ed6d1c4d2a9e7920754c960483 2024-12-08T03:58:59,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/38c4b2359d2d4940a327d1b1f939ba88 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/38c4b2359d2d4940a327d1b1f939ba88 2024-12-08T03:58:59,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/e183398870ef479cbf22ab97005058cb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/e183398870ef479cbf22ab97005058cb 2024-12-08T03:58:59,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/1c1ad4dea93748708bf4cc265af87880 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/1c1ad4dea93748708bf4cc265af87880 2024-12-08T03:58:59,471 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/recovered.edits/483.seqid, newMaxSeqId=483, maxSeqId=1 2024-12-08T03:58:59,472 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. 2024-12-08T03:58:59,472 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 97ee24ee195dd7bff50b82d61598e8ac: Waiting for close lock at 1733630338036Running coprocessor pre-close hooks at 1733630338036Disabling compacts and flushes for region at 1733630338036Disabling writes for close at 1733630338036Obtaining lock to block concurrent updates at 1733630338037 (+1 ms)Preparing flush snapshotting stores in 97ee24ee195dd7bff50b82d61598e8ac at 1733630338037Finished memstore snapshotting TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733630338037Flushing stores of TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac. at 1733630338038 (+1 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/A: creating writer at 1733630338038Flushing 97ee24ee195dd7bff50b82d61598e8ac/A: appending metadata at 1733630338043 (+5 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/A: closing flushed file at 1733630338043Flushing 97ee24ee195dd7bff50b82d61598e8ac/B: creating writer at 1733630338477 (+434 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/B: appending metadata at 1733630338483 (+6 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/B: closing flushed file at 1733630338484 (+1 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/C: creating writer at 1733630338901 (+417 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/C: appending metadata at 1733630338904 (+3 ms)Flushing 97ee24ee195dd7bff50b82d61598e8ac/C: closing flushed file at 1733630338904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a24b0b6: reopening flushed file at 1733630339319 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419b3de3: reopening flushed file at 1733630339325 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c418b57: reopening flushed file at 1733630339332 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 97ee24ee195dd7bff50b82d61598e8ac in 1303ms, sequenceid=480, compaction requested=true at 1733630339339 (+7 ms)Writing region close event to WAL at 1733630339467 (+128 ms)Running coprocessor post-close hooks at 1733630339471 (+4 ms)Closed at 1733630339472 (+1 ms) 2024-12-08T03:58:59,475 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:58:59,476 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=97ee24ee195dd7bff50b82d61598e8ac, regionState=CLOSED 2024-12-08T03:58:59,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:58:59,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T03:58:59,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 97ee24ee195dd7bff50b82d61598e8ac, server=32df4b4bb439,39919,1733630308479 in 1.6020 sec 2024-12-08T03:58:59,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T03:58:59,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97ee24ee195dd7bff50b82d61598e8ac, UNASSIGN in 1.6100 sec 2024-12-08T03:58:59,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-08T03:58:59,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6170 sec 2024-12-08T03:58:59,489 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630339489"}]},"ts":"1733630339489"} 2024-12-08T03:58:59,490 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:58:59,491 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:58:59,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6320 sec 2024-12-08T03:59:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T03:59:00,009 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:00,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:59:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,026 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T03:59:00,028 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:59:00,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/recovered.edits] 2024-12-08T03:59:00,041 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/23ea332466e042ffabccae807b7d600e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/23ea332466e042ffabccae807b7d600e 2024-12-08T03:59:00,042 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/454bc680c2094c448e7e55bdbe7db24a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/454bc680c2094c448e7e55bdbe7db24a 2024-12-08T03:59:00,044 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9e67e1fdbd034ef5aaa7ba859157adf5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/A/9e67e1fdbd034ef5aaa7ba859157adf5 2024-12-08T03:59:00,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0dcd0b2860734b199762de251445945f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/0dcd0b2860734b199762de251445945f 2024-12-08T03:59:00,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/637a1ccf4cbe4968a8e2c4d02af60f6a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/637a1ccf4cbe4968a8e2c4d02af60f6a 2024-12-08T03:59:00,050 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/dbf84c83b20d41068784a56cd8a59aae to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/B/dbf84c83b20d41068784a56cd8a59aae 2024-12-08T03:59:00,053 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6e7867bf4a8948ff8db20b297afa56e4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/6e7867bf4a8948ff8db20b297afa56e4 2024-12-08T03:59:00,054 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7319c2d15d334bc9ac38fc264f43a658 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/7319c2d15d334bc9ac38fc264f43a658 2024-12-08T03:59:00,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/cd5e579d9888434696776140f903333e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/C/cd5e579d9888434696776140f903333e 2024-12-08T03:59:00,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/recovered.edits/483.seqid to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac/recovered.edits/483.seqid 2024-12-08T03:59:00,059 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/97ee24ee195dd7bff50b82d61598e8ac 2024-12-08T03:59:00,059 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:59:00,063 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-08T03:59:00,072 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:59:00,096 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:59:00,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:59:00,098 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733630340097"}]},"ts":"9223372036854775807"} 2024-12-08T03:59:00,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:59:00,102 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 97ee24ee195dd7bff50b82d61598e8ac, NAME => 'TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:59:00,102 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:59:00,102 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733630340102"}]},"ts":"9223372036854775807"} 2024-12-08T03:59:00,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:59:00,107 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-12-08T03:59:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T03:59:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:59:00,138 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:00,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;32df4b4bb439:39919-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1543657143_22 at /127.0.0.1:57436 [Waiting for operation #315] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1907421056_22 at /127.0.0.1:53142 [Waiting for operation #332] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=377 (was 382), ProcessCount=11 (was 11), AvailableMemoryMB=16686 (was 17165) 2024-12-08T03:59:00,160 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=377, ProcessCount=11, AvailableMemoryMB=16686 2024-12-08T03:59:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:59:00,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:59:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:00,164 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:59:00,164 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:00,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-08T03:59:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T03:59:00,165 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:59:00,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741939_1115 (size=1061) 2024-12-08T03:59:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T03:59:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T03:59:00,577 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T03:59:00,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741940_1116 (size=53) 2024-12-08T03:59:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T03:59:00,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:59:00,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b7a5c1e2b432cf09bf0850d10fef26a2, disabling compactions & flushes 2024-12-08T03:59:00,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:00,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:00,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. after waiting 0 ms 2024-12-08T03:59:00,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:00,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:00,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7a5c1e2b432cf09bf0850d10fef26a2: Waiting for close lock at 1733630340987Disabling compacts and flushes for region at 1733630340987Disabling writes for close at 1733630340987Writing region close event to WAL at 1733630340987Closed at 1733630340987 2024-12-08T03:59:00,989 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:59:00,989 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733630340989"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630340989"}]},"ts":"1733630340989"} 2024-12-08T03:59:00,991 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:59:00,992 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:59:00,992 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630340992"}]},"ts":"1733630340992"} 2024-12-08T03:59:00,993 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:59:00,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, ASSIGN}] 2024-12-08T03:59:00,995 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, ASSIGN 2024-12-08T03:59:00,996 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, ASSIGN; state=OFFLINE, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=false 2024-12-08T03:59:01,147 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:01,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, ASSIGN because future has completed 2024-12-08T03:59:01,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:59:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T03:59:01,309 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:01,309 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:59:01,310 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,310 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:59:01,310 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,310 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,312 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,313 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:01,314 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a5c1e2b432cf09bf0850d10fef26a2 columnFamilyName A 2024-12-08T03:59:01,314 DEBUG [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:01,314 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(327): Store=b7a5c1e2b432cf09bf0850d10fef26a2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:01,315 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,316 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:01,316 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a5c1e2b432cf09bf0850d10fef26a2 columnFamilyName B 2024-12-08T03:59:01,317 DEBUG [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:01,317 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(327): Store=b7a5c1e2b432cf09bf0850d10fef26a2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:01,317 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,319 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:01,319 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a5c1e2b432cf09bf0850d10fef26a2 columnFamilyName C 2024-12-08T03:59:01,319 DEBUG [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:01,320 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(327): Store=b7a5c1e2b432cf09bf0850d10fef26a2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:01,320 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:01,320 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,322 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,322 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,323 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,323 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,324 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:59:01,325 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,327 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:59:01,327 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened b7a5c1e2b432cf09bf0850d10fef26a2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63214377, jitterRate=-0.05803237855434418}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:59:01,327 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:01,328 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for b7a5c1e2b432cf09bf0850d10fef26a2: Running coprocessor pre-open hook at 1733630341310Writing region info on filesystem at 1733630341310Initializing all the Stores at 1733630341311 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630341311Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630341311Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630341311Cleaning up temporary data from old regions at 1733630341323 (+12 ms)Running coprocessor post-open hooks at 1733630341327 (+4 ms)Region opened successfully at 1733630341328 (+1 ms) 2024-12-08T03:59:01,329 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., pid=34, masterSystemTime=1733630341306 2024-12-08T03:59:01,331 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:01,331 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:01,331 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:01,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:59:01,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-08T03:59:01,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 in 180 msec 2024-12-08T03:59:01,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T03:59:01,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, ASSIGN in 341 msec 2024-12-08T03:59:01,337 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:59:01,337 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630341337"}]},"ts":"1733630341337"} 2024-12-08T03:59:01,339 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:59:01,339 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:59:01,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-12-08T03:59:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T03:59:02,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:02,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:02,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:02,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:02,315 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279c0d05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:02,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:02,318 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:02,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:02,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:59:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T03:59:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bf6b1e5 2024-12-08T03:59:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:59:02,325 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:59:02,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:59:02,330 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:59:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:02,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741941_1117 (size=1097) 2024-12-08T03:59:02,751 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T03:59:02,751 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T03:59:02,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:59:02,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, REOPEN/MOVE}] 2024-12-08T03:59:02,769 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, REOPEN/MOVE 2024-12-08T03:59:02,769 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:02,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, REOPEN/MOVE because future has completed 2024-12-08T03:59:02,772 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:59:02,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:59:02,925 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:02,925 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:59:02,926 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing b7a5c1e2b432cf09bf0850d10fef26a2, disabling compactions & flushes 2024-12-08T03:59:02,926 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:02,926 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:02,926 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. after waiting 0 ms 2024-12-08T03:59:02,926 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:02,934 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T03:59:02,935 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:02,935 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for b7a5c1e2b432cf09bf0850d10fef26a2: Waiting for close lock at 1733630342926Running coprocessor pre-close hooks at 1733630342926Disabling compacts and flushes for region at 1733630342926Disabling writes for close at 1733630342926Writing region close event to WAL at 1733630342926Running coprocessor post-close hooks at 1733630342935 (+9 ms)Closed at 1733630342935 2024-12-08T03:59:02,935 WARN [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: b7a5c1e2b432cf09bf0850d10fef26a2 to self. 2024-12-08T03:59:02,939 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:02,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=CLOSED 2024-12-08T03:59:02,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:59:02,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-08T03:59:02,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 in 171 msec 2024-12-08T03:59:02,947 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=true 2024-12-08T03:59:03,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, REOPEN/MOVE because future has completed 2024-12-08T03:59:03,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:59:03,260 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,260 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:59:03,260 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,260 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:59:03,261 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,261 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,264 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,265 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:03,271 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a5c1e2b432cf09bf0850d10fef26a2 columnFamilyName A 2024-12-08T03:59:03,272 DEBUG [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:03,273 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(327): Store=b7a5c1e2b432cf09bf0850d10fef26a2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:03,273 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,274 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:03,274 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a5c1e2b432cf09bf0850d10fef26a2 columnFamilyName B 2024-12-08T03:59:03,274 DEBUG [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:03,275 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(327): Store=b7a5c1e2b432cf09bf0850d10fef26a2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:03,275 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,275 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:03,276 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7a5c1e2b432cf09bf0850d10fef26a2 columnFamilyName C 2024-12-08T03:59:03,276 DEBUG [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:03,276 INFO [StoreOpener-b7a5c1e2b432cf09bf0850d10fef26a2-1 {}] regionserver.HStore(327): Store=b7a5c1e2b432cf09bf0850d10fef26a2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:03,276 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,276 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,277 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,278 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,279 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,279 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,279 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:59:03,281 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,281 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened b7a5c1e2b432cf09bf0850d10fef26a2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60942447, jitterRate=-0.09188677370548248}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:59:03,282 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,282 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for b7a5c1e2b432cf09bf0850d10fef26a2: Running coprocessor pre-open hook at 1733630343261Writing region info on filesystem at 1733630343261Initializing all the Stores at 1733630343261Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630343262 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630343262Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630343262Cleaning up temporary data from old regions at 1733630343279 (+17 ms)Running coprocessor post-open hooks at 1733630343282 (+3 ms)Region opened successfully at 1733630343282 2024-12-08T03:59:03,283 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., pid=39, masterSystemTime=1733630343257 2024-12-08T03:59:03,285 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,285 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,286 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=OPEN, openSeqNum=5, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:59:03,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-08T03:59:03,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 in 187 msec 2024-12-08T03:59:03,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-08T03:59:03,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, REOPEN/MOVE in 524 msec 2024-12-08T03:59:03,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-08T03:59:03,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-12-08T03:59:03,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-12-08T03:59:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-08T03:59:03,300 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:03,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,301 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b58318f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,303 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3368167d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32abbac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b3d60a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3776d157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232b2d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30987bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f3edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15666438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81b325a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b09452b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27005e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,325 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e409a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,327 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0ba3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,328 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7829860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:03,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:03,330 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336c9523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:03,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14626b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:03,333 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,333 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,333 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,334 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-08T03:59:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,336 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,337 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,337 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:59:03,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,338 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:03,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:03,338 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,338 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T03:59:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:03,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:03,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:03,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:03,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:03,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:03,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:03,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084baeb1fbba454f99a177b40c49d0c2a8_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630343351/Put/seqid=0 2024-12-08T03:59:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:59:03,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741942_1118 (size=14594) 2024-12-08T03:59:03,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630343433, max timestamp=1733630343466Num uniques 859; 2024-12-08T03:59:03,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:03,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630343433, max timestamp=1733630343466Num uniques 859; 2024-12-08T03:59:03,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:03,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630343433, max timestamp=1733630343466Num uniques 859; 2024-12-08T03:59:03,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:03,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:03,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084baeb1fbba454f99a177b40c49d0c2a8_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084baeb1fbba454f99a177b40c49d0c2a8_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:03,481 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/817ed00af7a342018cf4c3cc29490c4a, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:03,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/817ed00af7a342018cf4c3cc29490c4a is 175, key is test_row_0/A:col10/1733630343351/Put/seqid=0 2024-12-08T03:59:03,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:59:03,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741943_1119 (size=39549) 2024-12-08T03:59:03,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630403515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630403517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630403519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630403519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630403520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630403627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630403627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630403638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630403638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630403638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:59:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:59:03,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:59:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630403837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630403837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630403847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630403857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630403857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:03,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:03,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/817ed00af7a342018cf4c3cc29490c4a 2024-12-08T03:59:03,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:59:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:03,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:03,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/039b06e5b82f4538b098c40e58bc0daa is 50, key is test_row_0/B:col10/1733630343351/Put/seqid=0 2024-12-08T03:59:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:59:03,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741944_1120 (size=12001) 2024-12-08T03:59:03,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/039b06e5b82f4538b098c40e58bc0daa 2024-12-08T03:59:04,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/37e883716c1043fab58431c590389c57 is 50, key is test_row_0/C:col10/1733630343351/Put/seqid=0 2024-12-08T03:59:04,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741945_1121 (size=12001) 2024-12-08T03:59:04,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/37e883716c1043fab58431c590389c57 2024-12-08T03:59:04,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/817ed00af7a342018cf4c3cc29490c4a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a 2024-12-08T03:59:04,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a, entries=200, sequenceid=15, filesize=38.6 K 2024-12-08T03:59:04,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/039b06e5b82f4538b098c40e58bc0daa as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/039b06e5b82f4538b098c40e58bc0daa 2024-12-08T03:59:04,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/039b06e5b82f4538b098c40e58bc0daa, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:59:04,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/37e883716c1043fab58431c590389c57 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/37e883716c1043fab58431c590389c57 2024-12-08T03:59:04,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/37e883716c1043fab58431c590389c57, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T03:59:04,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for b7a5c1e2b432cf09bf0850d10fef26a2 in 672ms, sequenceid=15, compaction requested=false 2024-12-08T03:59:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:04,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T03:59:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:04,111 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T03:59:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:04,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a426dcc322ae4d06b9b2eeb33716f6f2_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630343516/Put/seqid=0 2024-12-08T03:59:04,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741946_1122 (size=12154) 2024-12-08T03:59:04,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,149 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a426dcc322ae4d06b9b2eeb33716f6f2_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a426dcc322ae4d06b9b2eeb33716f6f2_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/41c8f89adfd74d7d89a6dae474f806bd, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:04,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/41c8f89adfd74d7d89a6dae474f806bd is 175, key is test_row_0/A:col10/1733630343516/Put/seqid=0 2024-12-08T03:59:04,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:04,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630404159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630404167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630404167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630404167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741947_1123 (size=30955) 2024-12-08T03:59:04,179 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/41c8f89adfd74d7d89a6dae474f806bd 2024-12-08T03:59:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630404177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/a81c1ee6eeef4340be7da87452e55b2b is 50, key is test_row_0/B:col10/1733630343516/Put/seqid=0 2024-12-08T03:59:04,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741948_1124 (size=12001) 2024-12-08T03:59:04,207 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/a81c1ee6eeef4340be7da87452e55b2b 2024-12-08T03:59:04,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/630d58f412d6418d99cdcef1d4b48900 is 50, key is test_row_0/C:col10/1733630343516/Put/seqid=0 2024-12-08T03:59:04,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741949_1125 (size=12001) 2024-12-08T03:59:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630404277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630404288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630404288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:59:04,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630404488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630404497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630404497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,638 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/630d58f412d6418d99cdcef1d4b48900 2024-12-08T03:59:04,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/41c8f89adfd74d7d89a6dae474f806bd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd 2024-12-08T03:59:04,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd, entries=150, sequenceid=52, filesize=30.2 K 2024-12-08T03:59:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/a81c1ee6eeef4340be7da87452e55b2b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a81c1ee6eeef4340be7da87452e55b2b 2024-12-08T03:59:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,657 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a81c1ee6eeef4340be7da87452e55b2b, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T03:59:04,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/630d58f412d6418d99cdcef1d4b48900 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/630d58f412d6418d99cdcef1d4b48900 2024-12-08T03:59:04,665 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/630d58f412d6418d99cdcef1d4b48900, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T03:59:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,666 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for b7a5c1e2b432cf09bf0850d10fef26a2 in 555ms, sequenceid=52, compaction requested=false 2024-12-08T03:59:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-08T03:59:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-08T03:59:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-08T03:59:04,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3300 sec 2024-12-08T03:59:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3380 sec 2024-12-08T03:59:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:04,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:04,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208845dcc0ef42146899d8e6203adadfcad_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741950_1126 (size=9714) 2024-12-08T03:59:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,758 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208845dcc0ef42146899d8e6203adadfcad_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208845dcc0ef42146899d8e6203adadfcad_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,759 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/aa8bd27238e24626b10525bfaecd8f07, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/aa8bd27238e24626b10525bfaecd8f07 is 175, key is test_row_0/A:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741951_1127 (size=22361) 2024-12-08T03:59:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/aa8bd27238e24626b10525bfaecd8f07 2024-12-08T03:59:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/dcae451cf2c7440290474c6a5823a29b is 50, key is test_row_0/B:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:04,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741952_1128 (size=9657) 2024-12-08T03:59:04,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/dcae451cf2c7440290474c6a5823a29b 2024-12-08T03:59:04,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630344774, max timestamp=1733630344806Num uniques 859; 2024-12-08T03:59:04,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630344774, max timestamp=1733630344806Num uniques 859; 2024-12-08T03:59:04,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630344774, max timestamp=1733630344806Num uniques 859; 2024-12-08T03:59:04,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:04,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630404806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:04,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/2778a321a982445ea200b6a89cbf703f is 50, key is test_row_0/C:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:04,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741953_1129 (size=9657) 2024-12-08T03:59:04,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/2778a321a982445ea200b6a89cbf703f 2024-12-08T03:59:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630404839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630404839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630404843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630404843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/aa8bd27238e24626b10525bfaecd8f07 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07 2024-12-08T03:59:04,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07, entries=100, sequenceid=62, filesize=21.8 K 2024-12-08T03:59:04,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/dcae451cf2c7440290474c6a5823a29b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/dcae451cf2c7440290474c6a5823a29b 2024-12-08T03:59:04,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/dcae451cf2c7440290474c6a5823a29b, entries=100, sequenceid=62, filesize=9.4 K 2024-12-08T03:59:04,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/2778a321a982445ea200b6a89cbf703f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2778a321a982445ea200b6a89cbf703f 2024-12-08T03:59:04,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2778a321a982445ea200b6a89cbf703f, entries=100, sequenceid=62, filesize=9.4 K 2024-12-08T03:59:04,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b7a5c1e2b432cf09bf0850d10fef26a2 in 165ms, sequenceid=62, compaction requested=true 2024-12-08T03:59:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:04,886 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:04,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:04,887 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:04,888 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T03:59:04,888 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:04,888 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:04,888 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=90.7 K 2024-12-08T03:59:04,888 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:04,889 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07] 2024-12-08T03:59:04,889 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:04,889 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:04,889 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:04,889 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/039b06e5b82f4538b098c40e58bc0daa, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a81c1ee6eeef4340be7da87452e55b2b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/dcae451cf2c7440290474c6a5823a29b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=32.9 K 2024-12-08T03:59:04,890 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 817ed00af7a342018cf4c3cc29490c4a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733630343351 2024-12-08T03:59:04,890 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 039b06e5b82f4538b098c40e58bc0daa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733630343351 2024-12-08T03:59:04,890 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c8f89adfd74d7d89a6dae474f806bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733630343514 2024-12-08T03:59:04,890 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a81c1ee6eeef4340be7da87452e55b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733630343514 2024-12-08T03:59:04,891 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8bd27238e24626b10525bfaecd8f07, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733630344164 2024-12-08T03:59:04,891 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting dcae451cf2c7440290474c6a5823a29b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733630344164 2024-12-08T03:59:04,908 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:04,910 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#111 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:04,911 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/53c6c8e785804ac7be6002af6eed4c73 is 50, key is test_row_0/B:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:04,919 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208a88cdc474c624b11bc1a9b797bf06ede_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:04,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-08T03:59:04,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:04,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:04,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:04,923 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208a88cdc474c624b11bc1a9b797bf06ede_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:04,924 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a88cdc474c624b11bc1a9b797bf06ede_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:04,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741954_1130 (size=12104) 2024-12-08T03:59:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630404947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630404947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630404954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630404958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630404958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:04,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ab18f57956534806be6d8cfd1b407602_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630344835/Put/seqid=0 2024-12-08T03:59:04,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741955_1131 (size=4469) 2024-12-08T03:59:04,988 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#112 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:04,989 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/bcab076596bb4ae2bc36ae19a4c6bce0 is 175, key is test_row_0/A:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:05,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741957_1133 (size=31165) 2024-12-08T03:59:05,002 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:59:05,009 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/bcab076596bb4ae2bc36ae19a4c6bce0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/bcab076596bb4ae2bc36ae19a4c6bce0 2024-12-08T03:59:05,017 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into bcab076596bb4ae2bc36ae19a4c6bce0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:05,017 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:05,017 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=13, startTime=1733630344886; duration=0sec 2024-12-08T03:59:05,017 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:05,017 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:05,017 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:05,021 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:05,021 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:05,021 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,021 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/37e883716c1043fab58431c590389c57, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/630d58f412d6418d99cdcef1d4b48900, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2778a321a982445ea200b6a89cbf703f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=32.9 K 2024-12-08T03:59:05,022 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e883716c1043fab58431c590389c57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733630343351 2024-12-08T03:59:05,023 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 630d58f412d6418d99cdcef1d4b48900, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733630343514 2024-12-08T03:59:05,023 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2778a321a982445ea200b6a89cbf703f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733630344164 2024-12-08T03:59:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741956_1132 (size=14594) 2024-12-08T03:59:05,043 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:05,044 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/492457bb8049493fbdc8e1b5e20e2d15 is 50, key is test_row_0/C:col10/1733630344719/Put/seqid=0 2024-12-08T03:59:05,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741958_1134 (size=12104) 2024-12-08T03:59:05,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630405057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,065 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/492457bb8049493fbdc8e1b5e20e2d15 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/492457bb8049493fbdc8e1b5e20e2d15 2024-12-08T03:59:05,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630405067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,075 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into 492457bb8049493fbdc8e1b5e20e2d15(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:05,075 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:05,075 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=13, startTime=1733630344887; duration=0sec 2024-12-08T03:59:05,075 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:05,076 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630405157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630405167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630405168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630405267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630405277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,342 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/53c6c8e785804ac7be6002af6eed4c73 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/53c6c8e785804ac7be6002af6eed4c73 2024-12-08T03:59:05,350 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into 53c6c8e785804ac7be6002af6eed4c73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:05,350 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:05,351 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=13, startTime=1733630344886; duration=0sec 2024-12-08T03:59:05,351 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:05,351 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:05,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:05,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ab18f57956534806be6d8cfd1b407602_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ab18f57956534806be6d8cfd1b407602_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:05,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/5317b9290bb344f2906e4b008ea2e5fb, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:05,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/5317b9290bb344f2906e4b008ea2e5fb is 175, key is test_row_0/A:col10/1733630344835/Put/seqid=0 2024-12-08T03:59:05,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741959_1135 (size=39549) 2024-12-08T03:59:05,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630405467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630405477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630405478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T03:59:05,487 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:05,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-08T03:59:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:59:05,490 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:05,490 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:05,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630405577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630405587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:59:05,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:59:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:59:05,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:05,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:59:05,846 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/5317b9290bb344f2906e4b008ea2e5fb 2024-12-08T03:59:05,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/d33e887bbd354a7c984c5c61192f338c is 50, key is test_row_0/B:col10/1733630344835/Put/seqid=0 2024-12-08T03:59:05,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741960_1136 (size=12001) 2024-12-08T03:59:05,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:59:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:05,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:05,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:05,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630405977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630405988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630405988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630406087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:06,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:06,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630406097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:06,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:59:06,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:06,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:06,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:06,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:06,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:59:06,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:59:06,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:06,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:06,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:06,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:06,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:06,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/d33e887bbd354a7c984c5c61192f338c 2024-12-08T03:59:06,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/a508799def664e90a00ba91b7915f952 is 50, key is test_row_0/C:col10/1733630344835/Put/seqid=0 2024-12-08T03:59:06,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741961_1137 (size=12001) 2024-12-08T03:59:06,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/a508799def664e90a00ba91b7915f952 2024-12-08T03:59:06,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/5317b9290bb344f2906e4b008ea2e5fb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb 2024-12-08T03:59:06,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb, entries=200, sequenceid=99, filesize=38.6 K 2024-12-08T03:59:06,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/d33e887bbd354a7c984c5c61192f338c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/d33e887bbd354a7c984c5c61192f338c 2024-12-08T03:59:06,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/d33e887bbd354a7c984c5c61192f338c, entries=150, sequenceid=99, filesize=11.7 K 2024-12-08T03:59:06,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/a508799def664e90a00ba91b7915f952 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/a508799def664e90a00ba91b7915f952 2024-12-08T03:59:06,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/a508799def664e90a00ba91b7915f952, entries=150, sequenceid=99, filesize=11.7 K 2024-12-08T03:59:06,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1424ms, sequenceid=99, compaction requested=false 2024-12-08T03:59:06,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:06,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:06,409 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:06,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:06,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f9e478332eb6483a83e855ad1fcf543a_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630344928/Put/seqid=0 2024-12-08T03:59:06,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741962_1138 (size=12154) 2024-12-08T03:59:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:59:06,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,834 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f9e478332eb6483a83e855ad1fcf543a_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f9e478332eb6483a83e855ad1fcf543a_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:06,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/df5cac304eb54ec09172b3cbe8a64ee5, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/df5cac304eb54ec09172b3cbe8a64ee5 is 175, key is test_row_0/A:col10/1733630344928/Put/seqid=0 2024-12-08T03:59:06,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741963_1139 (size=30955) 2024-12-08T03:59:06,840 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/df5cac304eb54ec09172b3cbe8a64ee5 2024-12-08T03:59:06,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/878a064367ba4f67900b8386789d6b79 is 50, key is test_row_0/B:col10/1733630344928/Put/seqid=0 2024-12-08T03:59:06,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741964_1140 (size=12001) 2024-12-08T03:59:06,868 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/878a064367ba4f67900b8386789d6b79 2024-12-08T03:59:06,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/e35678e6daab497f9d4dff55417c4c0f is 50, key is test_row_0/C:col10/1733630344928/Put/seqid=0 2024-12-08T03:59:06,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741965_1141 (size=12001) 2024-12-08T03:59:06,896 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/e35678e6daab497f9d4dff55417c4c0f 2024-12-08T03:59:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/df5cac304eb54ec09172b3cbe8a64ee5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5 2024-12-08T03:59:06,914 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5, entries=150, sequenceid=110, filesize=30.2 K 2024-12-08T03:59:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/878a064367ba4f67900b8386789d6b79 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/878a064367ba4f67900b8386789d6b79 2024-12-08T03:59:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,921 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/878a064367ba4f67900b8386789d6b79, entries=150, sequenceid=110, filesize=11.7 K 2024-12-08T03:59:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/e35678e6daab497f9d4dff55417c4c0f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e35678e6daab497f9d4dff55417c4c0f 2024-12-08T03:59:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,935 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e35678e6daab497f9d4dff55417c4c0f, entries=150, sequenceid=110, filesize=11.7 K 2024-12-08T03:59:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,936 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b7a5c1e2b432cf09bf0850d10fef26a2 in 527ms, sequenceid=110, compaction requested=true 2024-12-08T03:59:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-08T03:59:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-08T03:59:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-08T03:59:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-12-08T03:59:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4530 sec 2024-12-08T03:59:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:07,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:07,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ec65e9c56c7243e9917c222e87334bbd_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741966_1142 (size=21918) 2024-12-08T03:59:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ec65e9c56c7243e9917c222e87334bbd_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ec65e9c56c7243e9917c222e87334bbd_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/01baa4fb16a449448ecf777ae001d94c, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/01baa4fb16a449448ecf777ae001d94c is 175, key is test_row_0/A:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741967_1143 (size=65319) 2024-12-08T03:59:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:07,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630347107, max timestamp=1733630347137Num uniques 859; 2024-12-08T03:59:07,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630347107, max timestamp=1733630347137Num uniques 859; 2024-12-08T03:59:07,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:07,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:07,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630347107, max timestamp=1733630347137Num uniques 859; 2024-12-08T03:59:07,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630407154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630407155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630407156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630407157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630407160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630407267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630407268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630407268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630407268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630407277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630407477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630407477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630407477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630407477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630407487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/01baa4fb16a449448ecf777ae001d94c 2024-12-08T03:59:07,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/b8f6c317aa9c44e0aa8de275295e7357 is 50, key is test_row_0/B:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:07,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741968_1144 (size=9657) 2024-12-08T03:59:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T03:59:07,637 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:07,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-08T03:59:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:59:07,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:07,640 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:07,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:59:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630407787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630407787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630407787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630407787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:59:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:07,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:07,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:07,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630407797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:07,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/b8f6c317aa9c44e0aa8de275295e7357 2024-12-08T03:59:07,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/9c63d6a8654240c4a68beb4adc3f6e13 is 50, key is test_row_0/C:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:07,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741969_1145 (size=9657) 2024-12-08T03:59:07,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:59:07,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:07,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:07,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:07,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:07,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:59:08,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:59:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:08,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:59:08,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:08,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:08,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:59:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630408297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630408307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630408307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630408307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:08,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630408307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:08,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/9c63d6a8654240c4a68beb4adc3f6e13 2024-12-08T03:59:08,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/01baa4fb16a449448ecf777ae001d94c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c 2024-12-08T03:59:08,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c, entries=350, sequenceid=120, filesize=63.8 K 2024-12-08T03:59:08,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/b8f6c317aa9c44e0aa8de275295e7357 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/b8f6c317aa9c44e0aa8de275295e7357 2024-12-08T03:59:08,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/b8f6c317aa9c44e0aa8de275295e7357, entries=100, sequenceid=120, filesize=9.4 K 2024-12-08T03:59:08,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/9c63d6a8654240c4a68beb4adc3f6e13 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9c63d6a8654240c4a68beb4adc3f6e13 2024-12-08T03:59:08,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9c63d6a8654240c4a68beb4adc3f6e13, entries=100, sequenceid=120, filesize=9.4 K 2024-12-08T03:59:08,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1301ms, sequenceid=120, compaction requested=true 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:08,358 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:08,358 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:08,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:08,362 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:08,363 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:08,363 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,363 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:08,363 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:08,363 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/53c6c8e785804ac7be6002af6eed4c73, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/d33e887bbd354a7c984c5c61192f338c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/878a064367ba4f67900b8386789d6b79, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/b8f6c317aa9c44e0aa8de275295e7357] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=44.7 K 2024-12-08T03:59:08,363 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,363 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/bcab076596bb4ae2bc36ae19a4c6bce0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=163.1 K 2024-12-08T03:59:08,363 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,363 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/bcab076596bb4ae2bc36ae19a4c6bce0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c] 2024-12-08T03:59:08,364 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcab076596bb4ae2bc36ae19a4c6bce0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733630343516 2024-12-08T03:59:08,364 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c6c8e785804ac7be6002af6eed4c73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733630343516 2024-12-08T03:59:08,365 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d33e887bbd354a7c984c5c61192f338c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733630344825 2024-12-08T03:59:08,367 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5317b9290bb344f2906e4b008ea2e5fb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733630344825 2024-12-08T03:59:08,367 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 878a064367ba4f67900b8386789d6b79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733630344928 2024-12-08T03:59:08,367 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5cac304eb54ec09172b3cbe8a64ee5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733630344928 2024-12-08T03:59:08,367 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f6c317aa9c44e0aa8de275295e7357, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733630347053 2024-12-08T03:59:08,368 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01baa4fb16a449448ecf777ae001d94c, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733630347015 2024-12-08T03:59:08,385 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:08,386 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:08,387 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/00c69811a7464fdc881c5ff6e2540222 is 50, key is test_row_0/B:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:08,389 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208a3aef6b98fa848e7a410a8dec3707984_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:08,392 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208a3aef6b98fa848e7a410a8dec3707984_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:08,392 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a3aef6b98fa848e7a410a8dec3707984_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:08,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741970_1146 (size=12241) 2024-12-08T03:59:08,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741971_1147 (size=4469) 2024-12-08T03:59:08,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T03:59:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,407 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T03:59:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:08,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:08,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d4b28c4f7acd4a92a9224d5192b0a9f0_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630347154/Put/seqid=0 2024-12-08T03:59:08,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741972_1148 (size=12304) 2024-12-08T03:59:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:59:08,802 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/00c69811a7464fdc881c5ff6e2540222 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/00c69811a7464fdc881c5ff6e2540222 2024-12-08T03:59:08,803 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:08,804 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/e35c7cc40f9c4196b573654c508c7f8f is 175, key is test_row_0/A:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:08,811 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into 00c69811a7464fdc881c5ff6e2540222(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:08,811 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:08,811 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=12, startTime=1733630348358; duration=0sec 2024-12-08T03:59:08,812 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:08,812 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:08,812 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:08,813 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:08,813 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:08,813 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:08,814 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/492457bb8049493fbdc8e1b5e20e2d15, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/a508799def664e90a00ba91b7915f952, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e35678e6daab497f9d4dff55417c4c0f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9c63d6a8654240c4a68beb4adc3f6e13] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=44.7 K 2024-12-08T03:59:08,814 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 492457bb8049493fbdc8e1b5e20e2d15, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733630343516 2024-12-08T03:59:08,815 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a508799def664e90a00ba91b7915f952, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733630344825 2024-12-08T03:59:08,815 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e35678e6daab497f9d4dff55417c4c0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733630344928 2024-12-08T03:59:08,816 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c63d6a8654240c4a68beb4adc3f6e13, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733630347053 2024-12-08T03:59:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:08,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741973_1149 (size=31302) 2024-12-08T03:59:08,825 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d4b28c4f7acd4a92a9224d5192b0a9f0_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d4b28c4f7acd4a92a9224d5192b0a9f0_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:08,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/cb8c1c4bb67648768063c5c42b4c877e, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:08,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/cb8c1c4bb67648768063c5c42b4c877e is 175, key is test_row_0/A:col10/1733630347154/Put/seqid=0 2024-12-08T03:59:08,834 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:08,834 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/8d7b41c5af194788b047d53db1cefe9a is 50, key is test_row_0/C:col10/1733630347054/Put/seqid=0 2024-12-08T03:59:08,836 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/e35c7cc40f9c4196b573654c508c7f8f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e35c7cc40f9c4196b573654c508c7f8f 2024-12-08T03:59:08,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741974_1150 (size=31105) 2024-12-08T03:59:08,838 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/cb8c1c4bb67648768063c5c42b4c877e 2024-12-08T03:59:08,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741975_1151 (size=12241) 2024-12-08T03:59:08,851 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into e35c7cc40f9c4196b573654c508c7f8f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:08,851 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:08,851 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=12, startTime=1733630348358; duration=0sec 2024-12-08T03:59:08,851 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:08,851 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/35d6d03a7ab04de39ed0ee61916a075c is 50, key is test_row_0/B:col10/1733630347154/Put/seqid=0 2024-12-08T03:59:08,856 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/8d7b41c5af194788b047d53db1cefe9a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8d7b41c5af194788b047d53db1cefe9a 2024-12-08T03:59:08,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741976_1152 (size=12151) 2024-12-08T03:59:08,865 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into 8d7b41c5af194788b047d53db1cefe9a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:08,865 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:08,865 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=12, startTime=1733630348358; duration=0sec 2024-12-08T03:59:08,865 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:08,865 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:09,260 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/35d6d03a7ab04de39ed0ee61916a075c 2024-12-08T03:59:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/71d8538d28184f2aa9831316fd07e3f9 is 50, key is test_row_0/C:col10/1733630347154/Put/seqid=0 2024-12-08T03:59:09,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741977_1153 (size=12151) 2024-12-08T03:59:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:09,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630409324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630409324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630409325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630409325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630409327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,329 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630409437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630409437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630409437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630409438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630409647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630409647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630409647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630409648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:09,672 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/71d8538d28184f2aa9831316fd07e3f9 2024-12-08T03:59:09,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/cb8c1c4bb67648768063c5c42b4c877e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e 2024-12-08T03:59:09,681 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e, entries=150, sequenceid=155, filesize=30.4 K 2024-12-08T03:59:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/35d6d03a7ab04de39ed0ee61916a075c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/35d6d03a7ab04de39ed0ee61916a075c 2024-12-08T03:59:09,692 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/35d6d03a7ab04de39ed0ee61916a075c, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:59:09,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/71d8538d28184f2aa9831316fd07e3f9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/71d8538d28184f2aa9831316fd07e3f9 2024-12-08T03:59:09,702 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/71d8538d28184f2aa9831316fd07e3f9, entries=150, sequenceid=155, filesize=11.9 K 2024-12-08T03:59:09,703 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1296ms, sequenceid=155, compaction requested=false 2024-12-08T03:59:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-08T03:59:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-08T03:59:09,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-08T03:59:09,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0650 sec 2024-12-08T03:59:09,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0690 sec 2024-12-08T03:59:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T03:59:09,787 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:09,787 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-08T03:59:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:59:09,789 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:09,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:09,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:59:09,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:09,942 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:09,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084f5f8764145a4533b5e82d79cdc59c17_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630349318/Put/seqid=0 2024-12-08T03:59:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:09,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:09,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741978_1154 (size=12304) 2024-12-08T03:59:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:09,973 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084f5f8764145a4533b5e82d79cdc59c17_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084f5f8764145a4533b5e82d79cdc59c17_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/79b12dd825cd4c3d935613b8f5e4c888, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:09,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/79b12dd825cd4c3d935613b8f5e4c888 is 175, key is test_row_0/A:col10/1733630349318/Put/seqid=0 2024-12-08T03:59:09,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741979_1155 (size=31105) 2024-12-08T03:59:10,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630349972, max timestamp=1733630350008Num uniques 859; 2024-12-08T03:59:10,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630349972, max timestamp=1733630350008Num uniques 859; 2024-12-08T03:59:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630349972, max timestamp=1733630350008Num uniques 859; 2024-12-08T03:59:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:10,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630410024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630410027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630410032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630410033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:59:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630410137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630410137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630410146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630410146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630410347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630410347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630410357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630410358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,381 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/79b12dd825cd4c3d935613b8f5e4c888 2024-12-08T03:59:10,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/78529a94da5c4c48a4e33044a9729744 is 50, key is test_row_0/B:col10/1733630349318/Put/seqid=0 2024-12-08T03:59:10,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741980_1156 (size=12151) 2024-12-08T03:59:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:59:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630410657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630410657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630410667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630410668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:10,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:10,791 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/78529a94da5c4c48a4e33044a9729744 2024-12-08T03:59:10,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/c9dfa49351a849bd9d3e524ef4ab11e7 is 50, key is test_row_0/C:col10/1733630349318/Put/seqid=0 2024-12-08T03:59:10,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741981_1157 (size=12151) 2024-12-08T03:59:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:59:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630411167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630411167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630411177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630411177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,203 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/c9dfa49351a849bd9d3e524ef4ab11e7 2024-12-08T03:59:11,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/79b12dd825cd4c3d935613b8f5e4c888 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888 2024-12-08T03:59:11,213 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888, entries=150, sequenceid=169, filesize=30.4 K 2024-12-08T03:59:11,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/78529a94da5c4c48a4e33044a9729744 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/78529a94da5c4c48a4e33044a9729744 2024-12-08T03:59:11,218 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/78529a94da5c4c48a4e33044a9729744, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T03:59:11,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/c9dfa49351a849bd9d3e524ef4ab11e7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/c9dfa49351a849bd9d3e524ef4ab11e7 2024-12-08T03:59:11,223 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/c9dfa49351a849bd9d3e524ef4ab11e7, entries=150, sequenceid=169, filesize=11.9 K 2024-12-08T03:59:11,224 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1282ms, sequenceid=169, compaction requested=true 2024-12-08T03:59:11,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:11,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:11,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-08T03:59:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-08T03:59:11,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-08T03:59:11,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-12-08T03:59:11,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4390 sec 2024-12-08T03:59:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:11,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T03:59:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:11,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:11,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dcd0ac10c788485b85b62234ef85bdec_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:11,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741982_1158 (size=14794) 2024-12-08T03:59:11,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630411382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630411497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630411707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:11,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:11,773 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dcd0ac10c788485b85b62234ef85bdec_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dcd0ac10c788485b85b62234ef85bdec_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:11,774 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3c53a8f6176243948b6738edda90640e, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:11,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3c53a8f6176243948b6738edda90640e is 175, key is test_row_0/A:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:11,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741983_1159 (size=39749) 2024-12-08T03:59:11,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3c53a8f6176243948b6738edda90640e 2024-12-08T03:59:11,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/a458d12bd40e49d48fcd7c173ebc14eb is 50, key is test_row_0/B:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:11,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741984_1160 (size=12151) 2024-12-08T03:59:11,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/a458d12bd40e49d48fcd7c173ebc14eb 2024-12-08T03:59:11,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/13a8b9be2e3641a79d35b2090fdfba1a is 50, key is test_row_0/C:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:11,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741985_1161 (size=12151) 2024-12-08T03:59:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T03:59:11,947 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:11,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-08T03:59:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:59:11,950 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:11,951 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:11,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:12,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630412017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:59:12,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:59:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:12,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630412187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630412187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,188 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:12,188 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630412187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630412187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,192 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:12,192 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/13a8b9be2e3641a79d35b2090fdfba1a 2024-12-08T03:59:12,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3c53a8f6176243948b6738edda90640e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e 2024-12-08T03:59:12,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e, entries=200, sequenceid=204, filesize=38.8 K 2024-12-08T03:59:12,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/a458d12bd40e49d48fcd7c173ebc14eb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a458d12bd40e49d48fcd7c173ebc14eb 2024-12-08T03:59:12,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a458d12bd40e49d48fcd7c173ebc14eb, entries=150, sequenceid=204, filesize=11.9 K 2024-12-08T03:59:12,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/13a8b9be2e3641a79d35b2090fdfba1a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/13a8b9be2e3641a79d35b2090fdfba1a 2024-12-08T03:59:12,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/13a8b9be2e3641a79d35b2090fdfba1a, entries=150, sequenceid=204, filesize=11.9 K 2024-12-08T03:59:12,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b7a5c1e2b432cf09bf0850d10fef26a2 in 878ms, sequenceid=204, compaction requested=true 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:12,232 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T03:59:12,232 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:12,233 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:12,233 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:12,234 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,234 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/00c69811a7464fdc881c5ff6e2540222, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/35d6d03a7ab04de39ed0ee61916a075c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/78529a94da5c4c48a4e33044a9729744, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a458d12bd40e49d48fcd7c173ebc14eb] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=47.6 K 2024-12-08T03:59:12,234 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133261 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:12,234 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:12,234 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,234 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e35c7cc40f9c4196b573654c508c7f8f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=130.1 K 2024-12-08T03:59:12,234 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,234 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e35c7cc40f9c4196b573654c508c7f8f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e] 2024-12-08T03:59:12,234 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 00c69811a7464fdc881c5ff6e2540222, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733630344952 2024-12-08T03:59:12,234 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e35c7cc40f9c4196b573654c508c7f8f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733630344952 2024-12-08T03:59:12,234 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d6d03a7ab04de39ed0ee61916a075c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733630347154 2024-12-08T03:59:12,235 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb8c1c4bb67648768063c5c42b4c877e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733630347154 2024-12-08T03:59:12,235 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 78529a94da5c4c48a4e33044a9729744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733630349318 2024-12-08T03:59:12,235 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b12dd825cd4c3d935613b8f5e4c888, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733630349318 2024-12-08T03:59:12,235 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a458d12bd40e49d48fcd7c173ebc14eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733630350025 2024-12-08T03:59:12,235 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c53a8f6176243948b6738edda90640e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733630350020 2024-12-08T03:59:12,243 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:12,245 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:12,245 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/5a08ddc5d8624114b21438a454ff7e96 is 50, key is test_row_0/B:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:12,247 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208247ebbfa0eab47489f65e2941419c9e4_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:12,249 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208247ebbfa0eab47489f65e2941419c9e4_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:12,249 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208247ebbfa0eab47489f65e2941419c9e4_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:12,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741986_1162 (size=12527) 2024-12-08T03:59:12,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T03:59:12,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,258 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:12,263 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/5a08ddc5d8624114b21438a454ff7e96 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/5a08ddc5d8624114b21438a454ff7e96 2024-12-08T03:59:12,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741987_1163 (size=4469) 2024-12-08T03:59:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:59:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208052fb624509c45e2af152d0da49fe8ac_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630351381/Put/seqid=0 2024-12-08T03:59:12,273 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into 5a08ddc5d8624114b21438a454ff7e96(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:12,273 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#136 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:12,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:12,274 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=12, startTime=1733630352232; duration=0sec 2024-12-08T03:59:12,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:12,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:12,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:12,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/e998e36484684bdeb426f714e98241be is 175, key is test_row_0/A:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:12,277 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:12,277 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:12,277 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:12,277 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8d7b41c5af194788b047d53db1cefe9a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/71d8538d28184f2aa9831316fd07e3f9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/c9dfa49351a849bd9d3e524ef4ab11e7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/13a8b9be2e3641a79d35b2090fdfba1a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=47.6 K 2024-12-08T03:59:12,277 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7b41c5af194788b047d53db1cefe9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733630344952 2024-12-08T03:59:12,278 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d8538d28184f2aa9831316fd07e3f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733630347154 2024-12-08T03:59:12,278 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c9dfa49351a849bd9d3e524ef4ab11e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733630349318 2024-12-08T03:59:12,279 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a8b9be2e3641a79d35b2090fdfba1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733630350025 2024-12-08T03:59:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741988_1164 (size=12304) 2024-12-08T03:59:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:12,290 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:12,291 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/b463fc7ccbc040d2bc46255e83d5ce17 is 50, key is test_row_0/C:col10/1733630351350/Put/seqid=0 2024-12-08T03:59:12,293 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208052fb624509c45e2af152d0da49fe8ac_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208052fb624509c45e2af152d0da49fe8ac_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f35a641b858848c994d1a29b0396ed01, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:12,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f35a641b858848c994d1a29b0396ed01 is 175, key is test_row_0/A:col10/1733630351381/Put/seqid=0 2024-12-08T03:59:12,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741990_1166 (size=12527) 2024-12-08T03:59:12,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741989_1165 (size=31481) 2024-12-08T03:59:12,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741991_1167 (size=31105) 2024-12-08T03:59:12,328 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f35a641b858848c994d1a29b0396ed01 2024-12-08T03:59:12,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/c4810715f3364873a2ed2245a982967a is 50, key is test_row_0/B:col10/1733630351381/Put/seqid=0 2024-12-08T03:59:12,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741992_1168 (size=12151) 2024-12-08T03:59:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:12,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:59:12,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630352579, max timestamp=1733630352595Num uniques 859; 2024-12-08T03:59:12,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630352579, max timestamp=1733630352595Num uniques 859; 2024-12-08T03:59:12,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:12,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:12,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630352579, max timestamp=1733630352595Num uniques 859; 2024-12-08T03:59:12,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:12,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630412664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,702 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/b463fc7ccbc040d2bc46255e83d5ce17 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b463fc7ccbc040d2bc46255e83d5ce17 2024-12-08T03:59:12,711 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into b463fc7ccbc040d2bc46255e83d5ce17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:12,711 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:12,711 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=12, startTime=1733630352232; duration=0sec 2024-12-08T03:59:12,711 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:12,711 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:12,731 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/e998e36484684bdeb426f714e98241be as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e998e36484684bdeb426f714e98241be 2024-12-08T03:59:12,737 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into e998e36484684bdeb426f714e98241be(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:12,737 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:12,737 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=12, startTime=1733630352232; duration=0sec 2024-12-08T03:59:12,737 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:12,737 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:12,745 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/c4810715f3364873a2ed2245a982967a 2024-12-08T03:59:12,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/2f250eb750c44eeeb858869e1baeb761 is 50, key is test_row_0/C:col10/1733630351381/Put/seqid=0 2024-12-08T03:59:12,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741993_1169 (size=12151) 2024-12-08T03:59:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630412777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:12,849 INFO [master/32df4b4bb439:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T03:59:12,849 INFO [master/32df4b4bb439:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T03:59:12,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630412987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:59:13,157 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/2f250eb750c44eeeb858869e1baeb761 2024-12-08T03:59:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f35a641b858848c994d1a29b0396ed01 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01 2024-12-08T03:59:13,168 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01, entries=150, sequenceid=214, filesize=30.4 K 2024-12-08T03:59:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/c4810715f3364873a2ed2245a982967a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c4810715f3364873a2ed2245a982967a 2024-12-08T03:59:13,174 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c4810715f3364873a2ed2245a982967a, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T03:59:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/2f250eb750c44eeeb858869e1baeb761 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2f250eb750c44eeeb858869e1baeb761 2024-12-08T03:59:13,184 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2f250eb750c44eeeb858869e1baeb761, entries=150, sequenceid=214, filesize=11.9 K 2024-12-08T03:59:13,186 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for b7a5c1e2b432cf09bf0850d10fef26a2 in 927ms, sequenceid=214, compaction requested=false 2024-12-08T03:59:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-08T03:59:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-08T03:59:13,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-08T03:59:13,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2360 sec 2024-12-08T03:59:13,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2420 sec 2024-12-08T03:59:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:13,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-08T03:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:13,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c0bdf4dfa99341959104a2de59f261ce_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:13,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741994_1170 (size=14794) 2024-12-08T03:59:13,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630413326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630413437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630413647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:13,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:13,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c0bdf4dfa99341959104a2de59f261ce_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c0bdf4dfa99341959104a2de59f261ce_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:13,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/97105618f6ff4e5daf84db108aacdd31, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:13,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/97105618f6ff4e5daf84db108aacdd31 is 175, key is test_row_0/A:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:13,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741995_1171 (size=39749) 2024-12-08T03:59:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630413957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T03:59:14,097 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:14,097 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-08T03:59:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:59:14,099 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:14,099 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:14,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:14,132 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/97105618f6ff4e5daf84db108aacdd31 2024-12-08T03:59:14,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/c2dac7edac354c98aea386a2651712ae is 50, key is test_row_0/B:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:14,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741996_1172 (size=12151) 2024-12-08T03:59:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:59:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630414207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630414207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630414207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:14,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:14,207 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630414207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,208 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,208 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:14,208 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:14,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T03:59:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T03:59:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:59:14,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630414466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:14,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:14,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:14,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/c2dac7edac354c98aea386a2651712ae 2024-12-08T03:59:14,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/1d5e2a4fa32c4f77a5d35c74080e5f48 is 50, key is test_row_0/C:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:14,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741997_1173 (size=12151) 2024-12-08T03:59:14,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T03:59:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T03:59:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:59:14,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T03:59:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:14,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/1d5e2a4fa32c4f77a5d35c74080e5f48 2024-12-08T03:59:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/97105618f6ff4e5daf84db108aacdd31 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31 2024-12-08T03:59:14,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31, entries=200, sequenceid=254, filesize=38.8 K 2024-12-08T03:59:14,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/c2dac7edac354c98aea386a2651712ae as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c2dac7edac354c98aea386a2651712ae 2024-12-08T03:59:14,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c2dac7edac354c98aea386a2651712ae, entries=150, sequenceid=254, filesize=11.9 K 2024-12-08T03:59:14,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/1d5e2a4fa32c4f77a5d35c74080e5f48 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/1d5e2a4fa32c4f77a5d35c74080e5f48 2024-12-08T03:59:14,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/1d5e2a4fa32c4f77a5d35c74080e5f48, entries=150, sequenceid=254, filesize=11.9 K 2024-12-08T03:59:14,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1676ms, sequenceid=254, compaction requested=true 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:14,976 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:14,976 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:14,977 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:14,977 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:14,977 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:14,977 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:14,977 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,977 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,977 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/5a08ddc5d8624114b21438a454ff7e96, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c4810715f3364873a2ed2245a982967a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c2dac7edac354c98aea386a2651712ae] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.0 K 2024-12-08T03:59:14,977 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e998e36484684bdeb426f714e98241be, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=99.9 K 2024-12-08T03:59:14,978 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:14,978 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e998e36484684bdeb426f714e98241be, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31] 2024-12-08T03:59:14,978 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e998e36484684bdeb426f714e98241be, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733630350025 2024-12-08T03:59:14,978 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a08ddc5d8624114b21438a454ff7e96, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733630350025 2024-12-08T03:59:14,979 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c4810715f3364873a2ed2245a982967a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733630351374 2024-12-08T03:59:14,979 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f35a641b858848c994d1a29b0396ed01, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733630351374 2024-12-08T03:59:14,979 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97105618f6ff4e5daf84db108aacdd31, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733630352649 2024-12-08T03:59:14,979 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c2dac7edac354c98aea386a2651712ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733630352649 2024-12-08T03:59:14,986 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:14,986 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:14,986 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/785a2d5c7b1d42daa66066adf8277fc4 is 50, key is test_row_0/B:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:14,988 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208e5f185cfbf6d40438986ac2460fbe250_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:14,989 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208e5f185cfbf6d40438986ac2460fbe250_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:14,989 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e5f185cfbf6d40438986ac2460fbe250_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:15,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741998_1174 (size=12629) 2024-12-08T03:59:15,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741999_1175 (size=4469) 2024-12-08T03:59:15,006 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/785a2d5c7b1d42daa66066adf8277fc4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/785a2d5c7b1d42daa66066adf8277fc4 2024-12-08T03:59:15,010 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into 785a2d5c7b1d42daa66066adf8277fc4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:15,011 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:15,011 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=13, startTime=1733630354976; duration=0sec 2024-12-08T03:59:15,011 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:15,011 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:15,011 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:15,013 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:15,013 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:15,013 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:15,013 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b463fc7ccbc040d2bc46255e83d5ce17, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2f250eb750c44eeeb858869e1baeb761, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/1d5e2a4fa32c4f77a5d35c74080e5f48] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.0 K 2024-12-08T03:59:15,013 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b463fc7ccbc040d2bc46255e83d5ce17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733630350025 2024-12-08T03:59:15,014 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f250eb750c44eeeb858869e1baeb761, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733630351374 2024-12-08T03:59:15,014 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5e2a4fa32c4f77a5d35c74080e5f48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733630352649 2024-12-08T03:59:15,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:15,016 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:15,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b2bd94d6de7d4d8780de9868c6ea3326_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630353324/Put/seqid=0 2024-12-08T03:59:15,033 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:15,033 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/cb29adc122fd41a49de7a5843672e7da is 50, key is test_row_0/C:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742000_1176 (size=12454) 2024-12-08T03:59:15,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:15,040 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b2bd94d6de7d4d8780de9868c6ea3326_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b2bd94d6de7d4d8780de9868c6ea3326_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742001_1177 (size=12629) 2024-12-08T03:59:15,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f699fbde7dd345d0a5b42e56b0212df1, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f699fbde7dd345d0a5b42e56b0212df1 is 175, key is test_row_0/A:col10/1733630353324/Put/seqid=0 2024-12-08T03:59:15,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742002_1178 (size=31255) 2024-12-08T03:59:15,056 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f699fbde7dd345d0a5b42e56b0212df1 2024-12-08T03:59:15,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/3d22a3b567a1472aab5a2c523037d8dd is 50, key is test_row_0/B:col10/1733630353324/Put/seqid=0 2024-12-08T03:59:15,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742003_1179 (size=12301) 2024-12-08T03:59:15,069 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/3d22a3b567a1472aab5a2c523037d8dd 2024-12-08T03:59:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/9494d47b07d44532818389bfbf30d127 is 50, key is test_row_0/C:col10/1733630353324/Put/seqid=0 2024-12-08T03:59:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742004_1180 (size=12301) 2024-12-08T03:59:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:59:15,404 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#145 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:15,405 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/27451ed5845743ac9e106bc761596d4d is 175, key is test_row_0/A:col10/1733630352662/Put/seqid=0 2024-12-08T03:59:15,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742005_1181 (size=31583) 2024-12-08T03:59:15,451 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/cb29adc122fd41a49de7a5843672e7da as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/cb29adc122fd41a49de7a5843672e7da 2024-12-08T03:59:15,456 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into cb29adc122fd41a49de7a5843672e7da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:15,456 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:15,456 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=13, startTime=1733630354976; duration=0sec 2024-12-08T03:59:15,456 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:15,456 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:15,503 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/9494d47b07d44532818389bfbf30d127 2024-12-08T03:59:15,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/f699fbde7dd345d0a5b42e56b0212df1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1 2024-12-08T03:59:15,512 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1, entries=150, sequenceid=264, filesize=30.5 K 2024-12-08T03:59:15,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/3d22a3b567a1472aab5a2c523037d8dd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3d22a3b567a1472aab5a2c523037d8dd 2024-12-08T03:59:15,517 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3d22a3b567a1472aab5a2c523037d8dd, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T03:59:15,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/9494d47b07d44532818389bfbf30d127 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9494d47b07d44532818389bfbf30d127 2024-12-08T03:59:15,522 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9494d47b07d44532818389bfbf30d127, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T03:59:15,523 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=80.51 KB/82440 for b7a5c1e2b432cf09bf0850d10fef26a2 in 507ms, sequenceid=264, compaction requested=false 2024-12-08T03:59:15,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:15,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:15,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-08T03:59:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:15,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T03:59:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-08T03:59:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:15,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:15,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-08T03:59:15,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-12-08T03:59:15,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4310 sec 2024-12-08T03:59:15,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c3c4aeeba039422289f0a42e4ce33daa_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:15,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742006_1182 (size=12454) 2024-12-08T03:59:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630415570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630415677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:15,814 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/27451ed5845743ac9e106bc761596d4d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/27451ed5845743ac9e106bc761596d4d 2024-12-08T03:59:15,819 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into 27451ed5845743ac9e106bc761596d4d(size=30.8 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:15,819 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:15,819 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=13, startTime=1733630354976; duration=0sec 2024-12-08T03:59:15,820 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:15,820 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:15,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630415887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:15,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:15,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c3c4aeeba039422289f0a42e4ce33daa_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c3c4aeeba039422289f0a42e4ce33daa_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:15,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/da80669f7a514c9b8da8245438d0e19a, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:15,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/da80669f7a514c9b8da8245438d0e19a is 175, key is test_row_0/A:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:15,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742007_1183 (size=31255) 2024-12-08T03:59:15,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/da80669f7a514c9b8da8245438d0e19a 2024-12-08T03:59:15,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/2a92346571e54bf9b74b322e806cfc5c is 50, key is test_row_0/B:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:15,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742008_1184 (size=12301) 2024-12-08T03:59:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630416197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T03:59:16,247 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:16,247 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-08T03:59:16,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T03:59:16,250 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:16,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T03:59:16,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/2a92346571e54bf9b74b322e806cfc5c 2024-12-08T03:59:16,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/4702f2f4a15644ae8cb599de1ba29dbd is 50, key is test_row_0/C:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:16,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742009_1185 (size=12301) 2024-12-08T03:59:16,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T03:59:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T03:59:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T03:59:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630416707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:16,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T03:59:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:16,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/4702f2f4a15644ae8cb599de1ba29dbd 2024-12-08T03:59:16,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/da80669f7a514c9b8da8245438d0e19a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a 2024-12-08T03:59:16,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a, entries=150, sequenceid=281, filesize=30.5 K 2024-12-08T03:59:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/2a92346571e54bf9b74b322e806cfc5c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/2a92346571e54bf9b74b322e806cfc5c 2024-12-08T03:59:16,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/2a92346571e54bf9b74b322e806cfc5c, entries=150, sequenceid=281, filesize=12.0 K 2024-12-08T03:59:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/4702f2f4a15644ae8cb599de1ba29dbd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/4702f2f4a15644ae8cb599de1ba29dbd 2024-12-08T03:59:16,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/4702f2f4a15644ae8cb599de1ba29dbd, entries=150, sequenceid=281, filesize=12.0 K 2024-12-08T03:59:16,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1290ms, sequenceid=281, compaction requested=true 2024-12-08T03:59:16,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:16,815 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:16,815 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:16,816 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,816 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,816 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/27451ed5845743ac9e106bc761596d4d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=91.9 K 2024-12-08T03:59:16,816 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/785a2d5c7b1d42daa66066adf8277fc4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3d22a3b567a1472aab5a2c523037d8dd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/2a92346571e54bf9b74b322e806cfc5c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.4 K 2024-12-08T03:59:16,816 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/27451ed5845743ac9e106bc761596d4d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a] 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 785a2d5c7b1d42daa66066adf8277fc4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733630352649 2024-12-08T03:59:16,816 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27451ed5845743ac9e106bc761596d4d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733630352649 2024-12-08T03:59:16,817 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d22a3b567a1472aab5a2c523037d8dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733630353303 2024-12-08T03:59:16,817 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f699fbde7dd345d0a5b42e56b0212df1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733630353303 2024-12-08T03:59:16,817 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting da80669f7a514c9b8da8245438d0e19a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733630355513 2024-12-08T03:59:16,817 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a92346571e54bf9b74b322e806cfc5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733630355513 2024-12-08T03:59:16,823 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:16,828 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120840fe4cdd219d4558959d1fe86704e9de_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:16,829 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:16,829 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/bf91287aeae04dd0b68e5681e33c021f is 50, key is test_row_0/B:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:16,830 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120840fe4cdd219d4558959d1fe86704e9de_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:16,830 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120840fe4cdd219d4558959d1fe86704e9de_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:16,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742010_1186 (size=12881) 2024-12-08T03:59:16,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742011_1187 (size=4469) 2024-12-08T03:59:16,839 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/bf91287aeae04dd0b68e5681e33c021f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/bf91287aeae04dd0b68e5681e33c021f 2024-12-08T03:59:16,844 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into bf91287aeae04dd0b68e5681e33c021f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:16,844 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:16,844 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=13, startTime=1733630356815; duration=0sec 2024-12-08T03:59:16,844 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:16,844 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:16,844 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:16,845 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:16,846 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:16,846 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,846 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/cb29adc122fd41a49de7a5843672e7da, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9494d47b07d44532818389bfbf30d127, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/4702f2f4a15644ae8cb599de1ba29dbd] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.4 K 2024-12-08T03:59:16,846 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting cb29adc122fd41a49de7a5843672e7da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733630352649 2024-12-08T03:59:16,846 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 9494d47b07d44532818389bfbf30d127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733630353303 2024-12-08T03:59:16,846 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4702f2f4a15644ae8cb599de1ba29dbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733630355513 2024-12-08T03:59:16,854 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:16,855 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/755636c7d5734f50a0f7215f11e26330 is 50, key is test_row_0/C:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:16,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742012_1188 (size=12881) 2024-12-08T03:59:16,862 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/755636c7d5734f50a0f7215f11e26330 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/755636c7d5734f50a0f7215f11e26330 2024-12-08T03:59:16,867 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into 755636c7d5734f50a0f7215f11e26330(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:16,867 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:16,867 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=13, startTime=1733630356815; duration=0sec 2024-12-08T03:59:16,867 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:16,867 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:16,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T03:59:16,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:16,877 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T03:59:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T03:59:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082710c669ddce4f7f8972487cb68ac8cd_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630355531/Put/seqid=0 2024-12-08T03:59:16,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742013_1189 (size=12454) 2024-12-08T03:59:17,239 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:17,239 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/adc13df5f3934ab58bc432910b58545d is 175, key is test_row_0/A:col10/1733630355518/Put/seqid=0 2024-12-08T03:59:17,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742014_1190 (size=31835) 2024-12-08T03:59:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:17,292 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082710c669ddce4f7f8972487cb68ac8cd_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082710c669ddce4f7f8972487cb68ac8cd_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/2a271e9e5e4d4fcb84c057cc06b57ce0, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/2a271e9e5e4d4fcb84c057cc06b57ce0 is 175, key is test_row_0/A:col10/1733630355531/Put/seqid=0 2024-12-08T03:59:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742015_1191 (size=31255) 2024-12-08T03:59:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T03:59:17,649 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/adc13df5f3934ab58bc432910b58545d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/adc13df5f3934ab58bc432910b58545d 2024-12-08T03:59:17,655 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into adc13df5f3934ab58bc432910b58545d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:17,655 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:17,655 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=13, startTime=1733630356815; duration=0sec 2024-12-08T03:59:17,657 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:17,657 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:17,699 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/2a271e9e5e4d4fcb84c057cc06b57ce0 2024-12-08T03:59:17,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/6dbcacc2b8c94d95b7ccfa2c89026278 is 50, key is test_row_0/B:col10/1733630355531/Put/seqid=0 2024-12-08T03:59:17,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742016_1192 (size=12301) 2024-12-08T03:59:17,716 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/6dbcacc2b8c94d95b7ccfa2c89026278 2024-12-08T03:59:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:17,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/e6895f23947940889d1ebde79deadef2 is 50, key is test_row_0/C:col10/1733630355531/Put/seqid=0 2024-12-08T03:59:17,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742017_1193 (size=12301) 2024-12-08T03:59:17,740 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/e6895f23947940889d1ebde79deadef2 2024-12-08T03:59:17,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/2a271e9e5e4d4fcb84c057cc06b57ce0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0 2024-12-08T03:59:17,751 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0, entries=150, sequenceid=300, filesize=30.5 K 2024-12-08T03:59:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/6dbcacc2b8c94d95b7ccfa2c89026278 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/6dbcacc2b8c94d95b7ccfa2c89026278 2024-12-08T03:59:17,758 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/6dbcacc2b8c94d95b7ccfa2c89026278, entries=150, sequenceid=300, filesize=12.0 K 2024-12-08T03:59:17,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/e6895f23947940889d1ebde79deadef2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e6895f23947940889d1ebde79deadef2 2024-12-08T03:59:17,764 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e6895f23947940889d1ebde79deadef2, entries=150, sequenceid=300, filesize=12.0 K 2024-12-08T03:59:17,765 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7a5c1e2b432cf09bf0850d10fef26a2 in 888ms, sequenceid=300, compaction requested=false 2024-12-08T03:59:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-08T03:59:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-08T03:59:17,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-08T03:59:17,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-12-08T03:59:17,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5200 sec 2024-12-08T03:59:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:17,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T03:59:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:17,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:17,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086b4f419803a24476950c617cc693c233_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:17,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742018_1194 (size=14994) 2024-12-08T03:59:17,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630417812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630417927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630418137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:18,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086b4f419803a24476950c617cc693c233_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086b4f419803a24476950c617cc693c233_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:18,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/1913feef04164344b336dd1741064c4c, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:18,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/1913feef04164344b336dd1741064c4c is 175, key is test_row_0/A:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742019_1195 (size=39949) 2024-12-08T03:59:18,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58098 deadline: 1733630418227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58128 deadline: 1733630418227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,229 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:18,229 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58050 deadline: 1733630418237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,239 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58092 deadline: 1733630418247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,250 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in b7a5c1e2b432cf09bf0850d10fef26a2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T03:59:18,397 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:18,398 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-08T03:59:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:18,399 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:18,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:18,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:18,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630418446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:18,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:18,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/1913feef04164344b336dd1741064c4c 2024-12-08T03:59:18,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/4a63fadb0a994af1af529bd1037310ae is 50, key is test_row_0/B:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:18,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742020_1196 (size=12301) 2024-12-08T03:59:18,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:18,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:18,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:18,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:18,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630418957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:19,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/4a63fadb0a994af1af529bd1037310ae 2024-12-08T03:59:19,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:19,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:19,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/8288a653b87a48f8a4d7c5006bb3f62e is 50, key is test_row_0/C:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:19,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742021_1197 (size=12301) 2024-12-08T03:59:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:19,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:19,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:19,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:19,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:19,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:19,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/8288a653b87a48f8a4d7c5006bb3f62e 2024-12-08T03:59:19,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/1913feef04164344b336dd1741064c4c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c 2024-12-08T03:59:19,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c, entries=200, sequenceid=318, filesize=39.0 K 2024-12-08T03:59:19,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/4a63fadb0a994af1af529bd1037310ae as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4a63fadb0a994af1af529bd1037310ae 2024-12-08T03:59:19,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4a63fadb0a994af1af529bd1037310ae, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T03:59:19,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/8288a653b87a48f8a4d7c5006bb3f62e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8288a653b87a48f8a4d7c5006bb3f62e 2024-12-08T03:59:19,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8288a653b87a48f8a4d7c5006bb3f62e, entries=150, sequenceid=318, filesize=12.0 K 2024-12-08T03:59:19,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1671ms, sequenceid=318, compaction requested=true 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:19,441 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:19,441 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:19,442 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:19,442 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:19,442 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:19,442 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,442 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:19,442 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,443 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/bf91287aeae04dd0b68e5681e33c021f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/6dbcacc2b8c94d95b7ccfa2c89026278, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4a63fadb0a994af1af529bd1037310ae] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.6 K 2024-12-08T03:59:19,443 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/adc13df5f3934ab58bc432910b58545d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=100.6 K 2024-12-08T03:59:19,443 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,443 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/adc13df5f3934ab58bc432910b58545d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c] 2024-12-08T03:59:19,443 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting bf91287aeae04dd0b68e5681e33c021f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733630355513 2024-12-08T03:59:19,443 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting adc13df5f3934ab58bc432910b58545d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733630355513 2024-12-08T03:59:19,444 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dbcacc2b8c94d95b7ccfa2c89026278, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733630355531 2024-12-08T03:59:19,444 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a271e9e5e4d4fcb84c057cc06b57ce0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733630355531 2024-12-08T03:59:19,444 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1913feef04164344b336dd1741064c4c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733630357752 2024-12-08T03:59:19,444 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a63fadb0a994af1af529bd1037310ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733630357752 2024-12-08T03:59:19,451 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:19,453 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:19,454 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/84a45a31f17644399419aaaa44c12a28 is 50, key is test_row_0/B:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:19,459 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412088dc54e2090124d5a983ba6614bd1fdaa_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:19,460 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412088dc54e2090124d5a983ba6614bd1fdaa_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:19,460 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088dc54e2090124d5a983ba6614bd1fdaa_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:19,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742022_1198 (size=12983) 2024-12-08T03:59:19,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-08T03:59:19,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,469 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T03:59:19,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:19,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:19,471 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/84a45a31f17644399419aaaa44c12a28 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/84a45a31f17644399419aaaa44c12a28 2024-12-08T03:59:19,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742023_1199 (size=4469) 2024-12-08T03:59:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086f076bdaa5b8416aaf5f85b3a2b20ad3_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630357792/Put/seqid=0 2024-12-08T03:59:19,481 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into 84a45a31f17644399419aaaa44c12a28(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:19,482 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:19,482 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=13, startTime=1733630359441; duration=0sec 2024-12-08T03:59:19,482 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:19,482 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:19,482 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:19,483 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:19,483 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:19,484 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:19,484 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/755636c7d5734f50a0f7215f11e26330, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e6895f23947940889d1ebde79deadef2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8288a653b87a48f8a4d7c5006bb3f62e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.6 K 2024-12-08T03:59:19,484 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 755636c7d5734f50a0f7215f11e26330, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733630355513 2024-12-08T03:59:19,484 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e6895f23947940889d1ebde79deadef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733630355531 2024-12-08T03:59:19,485 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8288a653b87a48f8a4d7c5006bb3f62e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733630357752 2024-12-08T03:59:19,492 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:19,493 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/b041be4faafa4ac5b8ab27535d3de62c is 50, key is test_row_0/C:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:19,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742024_1200 (size=12454) 2024-12-08T03:59:19,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742025_1201 (size=12983) 2024-12-08T03:59:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:19,875 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:19,876 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/694715b111c54e63aaf541ff9530a829 is 175, key is test_row_0/A:col10/1733630357752/Put/seqid=0 2024-12-08T03:59:19,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742026_1202 (size=31937) 2024-12-08T03:59:19,884 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/694715b111c54e63aaf541ff9530a829 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/694715b111c54e63aaf541ff9530a829 2024-12-08T03:59:19,888 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into 694715b111c54e63aaf541ff9530a829(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:19,888 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:19,888 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=13, startTime=1733630359441; duration=0sec 2024-12-08T03:59:19,889 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:19,889 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:19,900 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086f076bdaa5b8416aaf5f85b3a2b20ad3_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086f076bdaa5b8416aaf5f85b3a2b20ad3_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:19,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3ecf886a31c64023917b09fd5c9e0fc9, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:19,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3ecf886a31c64023917b09fd5c9e0fc9 is 175, key is test_row_0/A:col10/1733630357792/Put/seqid=0 2024-12-08T03:59:19,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742027_1203 (size=31255) 2024-12-08T03:59:19,913 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/b041be4faafa4ac5b8ab27535d3de62c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b041be4faafa4ac5b8ab27535d3de62c 2024-12-08T03:59:19,918 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into b041be4faafa4ac5b8ab27535d3de62c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:19,918 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:19,918 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=13, startTime=1733630359441; duration=0sec 2024-12-08T03:59:19,918 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:19,918 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:19,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:20,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630420021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:20,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630420126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:20,306 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3ecf886a31c64023917b09fd5c9e0fc9 2024-12-08T03:59:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/28161b6ac15547d186b18d052fddf78d is 50, key is test_row_0/B:col10/1733630357792/Put/seqid=0 2024-12-08T03:59:20,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742028_1204 (size=12301) 2024-12-08T03:59:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630420337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:20,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630420647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:20,718 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/28161b6ac15547d186b18d052fddf78d 2024-12-08T03:59:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/3b0053c2f4804bc0acced259bdac66b8 is 50, key is test_row_0/C:col10/1733630357792/Put/seqid=0 2024-12-08T03:59:20,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742029_1205 (size=12301) 2024-12-08T03:59:21,141 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/3b0053c2f4804bc0acced259bdac66b8 2024-12-08T03:59:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/3ecf886a31c64023917b09fd5c9e0fc9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9 2024-12-08T03:59:21,150 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9, entries=150, sequenceid=333, filesize=30.5 K 2024-12-08T03:59:21,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/28161b6ac15547d186b18d052fddf78d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/28161b6ac15547d186b18d052fddf78d 2024-12-08T03:59:21,155 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/28161b6ac15547d186b18d052fddf78d, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T03:59:21,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/3b0053c2f4804bc0acced259bdac66b8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/3b0053c2f4804bc0acced259bdac66b8 2024-12-08T03:59:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630421157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:21,160 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/3b0053c2f4804bc0acced259bdac66b8, entries=150, sequenceid=333, filesize=12.0 K 2024-12-08T03:59:21,161 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1692ms, sequenceid=333, compaction requested=false 2024-12-08T03:59:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-08T03:59:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-08T03:59:21,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-08T03:59:21,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7630 sec 2024-12-08T03:59:21,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7670 sec 2024-12-08T03:59:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:22,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T03:59:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:22,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e9ef5566df9d458e9b0260d95bcc84fd_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:22,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742030_1206 (size=14994) 2024-12-08T03:59:22,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630422204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:22,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630422317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:22,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630422527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T03:59:22,578 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:22,578 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-08T03:59:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:22,580 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:22,580 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:22,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:22,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:22,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e9ef5566df9d458e9b0260d95bcc84fd_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e9ef5566df9d458e9b0260d95bcc84fd_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:22,595 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/7f8d44ff96e04b14bcacdd922deb1ac9, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:22,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/7f8d44ff96e04b14bcacdd922deb1ac9 is 175, key is test_row_0/A:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:22,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742031_1207 (size=39949) 2024-12-08T03:59:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:22,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:22,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:22,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630422837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:22,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:22,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:22,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/7f8d44ff96e04b14bcacdd922deb1ac9 2024-12-08T03:59:23,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/3fd6bdd3706b42e8b7343d7f746406c4 is 50, key is test_row_0/B:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:23,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742032_1208 (size=12301) 2024-12-08T03:59:23,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:23,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:23,338 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-12-08T03:59:23,338 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:23,338 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,338 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:23,340 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-12-08T03:59:23,340 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:23,340 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,341 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-12-08T03:59:23,341 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:23,341 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:23,341 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,341 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:23,342 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-12-08T03:59:23,342 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:23,342 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,342 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:23,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:23,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58040 deadline: 1733630423347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:23,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T03:59:23,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/3fd6bdd3706b42e8b7343d7f746406c4 2024-12-08T03:59:23,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/af9c83282f114b8aa63846e403fafd93 is 50, key is test_row_0/C:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742033_1209 (size=12301) 2024-12-08T03:59:23,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:23,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:23,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:23,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:23,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/af9c83282f114b8aa63846e403fafd93 2024-12-08T03:59:23,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/7f8d44ff96e04b14bcacdd922deb1ac9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9 2024-12-08T03:59:23,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9, entries=200, sequenceid=354, filesize=39.0 K 2024-12-08T03:59:23,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/3fd6bdd3706b42e8b7343d7f746406c4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3fd6bdd3706b42e8b7343d7f746406c4 2024-12-08T03:59:23,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3fd6bdd3706b42e8b7343d7f746406c4, entries=150, sequenceid=354, filesize=12.0 K 2024-12-08T03:59:23,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/af9c83282f114b8aa63846e403fafd93 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/af9c83282f114b8aa63846e403fafd93 2024-12-08T03:59:23,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/af9c83282f114b8aa63846e403fafd93, entries=150, sequenceid=354, filesize=12.0 K 2024-12-08T03:59:23,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1677ms, sequenceid=354, compaction requested=true 2024-12-08T03:59:23,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:23,847 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7a5c1e2b432cf09bf0850d10fef26a2:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:23,847 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:23,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:23,848 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:23,848 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:23,848 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/B is initiating minor compaction (all files) 2024-12-08T03:59:23,848 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/A is initiating minor compaction (all files) 2024-12-08T03:59:23,848 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/B in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,848 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/A in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,848 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/84a45a31f17644399419aaaa44c12a28, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/28161b6ac15547d186b18d052fddf78d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3fd6bdd3706b42e8b7343d7f746406c4] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.7 K 2024-12-08T03:59:23,848 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/694715b111c54e63aaf541ff9530a829, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=100.7 K 2024-12-08T03:59:23,848 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,848 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/694715b111c54e63aaf541ff9530a829, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9] 2024-12-08T03:59:23,848 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a45a31f17644399419aaaa44c12a28, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733630357752 2024-12-08T03:59:23,849 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 694715b111c54e63aaf541ff9530a829, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733630357752 2024-12-08T03:59:23,849 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 28161b6ac15547d186b18d052fddf78d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733630357792 2024-12-08T03:59:23,849 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ecf886a31c64023917b09fd5c9e0fc9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733630357792 2024-12-08T03:59:23,849 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd6bdd3706b42e8b7343d7f746406c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733630360016 2024-12-08T03:59:23,850 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8d44ff96e04b14bcacdd922deb1ac9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733630360008 2024-12-08T03:59:23,858 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:23,859 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/4dfc0eb2745c4093a2d892c5cacbf89b is 50, key is test_row_0/B:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:23,860 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:23,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742034_1210 (size=13085) 2024-12-08T03:59:23,862 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208842c6000ce8d45d7a371a751acfe5384_b7a5c1e2b432cf09bf0850d10fef26a2 store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:23,883 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208842c6000ce8d45d7a371a751acfe5384_b7a5c1e2b432cf09bf0850d10fef26a2, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:23,883 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208842c6000ce8d45d7a371a751acfe5384_b7a5c1e2b432cf09bf0850d10fef26a2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:23,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742035_1211 (size=4469) 2024-12-08T03:59:23,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-08T03:59:23,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:23,964 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T03:59:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a02c1c338bab425990f9aa0de4aba352_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630362203/Put/seqid=0 2024-12-08T03:59:23,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742036_1212 (size=12454) 2024-12-08T03:59:24,289 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:24,290 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/4dfc0eb2745c4093a2d892c5cacbf89b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4dfc0eb2745c4093a2d892c5cacbf89b 2024-12-08T03:59:24,291 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/1b8efaf88131452d97397da3a627de0a is 175, key is test_row_0/A:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:24,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742037_1213 (size=32039) 2024-12-08T03:59:24,299 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/B of b7a5c1e2b432cf09bf0850d10fef26a2 into 4dfc0eb2745c4093a2d892c5cacbf89b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:24,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:24,299 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/B, priority=13, startTime=1733630363847; duration=0sec 2024-12-08T03:59:24,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:24,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:B 2024-12-08T03:59:24,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:24,300 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:24,300 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): b7a5c1e2b432cf09bf0850d10fef26a2/C is initiating minor compaction (all files) 2024-12-08T03:59:24,300 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7a5c1e2b432cf09bf0850d10fef26a2/C in TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:24,300 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b041be4faafa4ac5b8ab27535d3de62c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/3b0053c2f4804bc0acced259bdac66b8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/af9c83282f114b8aa63846e403fafd93] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp, totalSize=36.7 K 2024-12-08T03:59:24,301 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b041be4faafa4ac5b8ab27535d3de62c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733630357752 2024-12-08T03:59:24,301 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0053c2f4804bc0acced259bdac66b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733630357792 2024-12-08T03:59:24,301 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting af9c83282f114b8aa63846e403fafd93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733630360016 2024-12-08T03:59:24,310 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7a5c1e2b432cf09bf0850d10fef26a2#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:24,311 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/b3348110858645ed8e0bef1bcc49515a is 50, key is test_row_0/C:col10/1733630360019/Put/seqid=0 2024-12-08T03:59:24,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742038_1214 (size=13085) 2024-12-08T03:59:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:24,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. as already flushing 2024-12-08T03:59:24,358 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-12-08T03:59:24,358 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:24,358 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:24,359 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:24,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:24,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:24,381 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a02c1c338bab425990f9aa0de4aba352_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a02c1c338bab425990f9aa0de4aba352_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:24,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/42360f1b35e04d409e5cf81d73a834dd, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:24,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/42360f1b35e04d409e5cf81d73a834dd is 175, key is test_row_0/A:col10/1733630362203/Put/seqid=0 2024-12-08T03:59:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742039_1215 (size=31255) 2024-12-08T03:59:24,703 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/1b8efaf88131452d97397da3a627de0a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1b8efaf88131452d97397da3a627de0a 2024-12-08T03:59:24,709 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/A of b7a5c1e2b432cf09bf0850d10fef26a2 into 1b8efaf88131452d97397da3a627de0a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:24,709 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:24,709 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/A, priority=13, startTime=1733630363847; duration=0sec 2024-12-08T03:59:24,709 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:24,709 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:A 2024-12-08T03:59:24,721 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/b3348110858645ed8e0bef1bcc49515a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b3348110858645ed8e0bef1bcc49515a 2024-12-08T03:59:24,726 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7a5c1e2b432cf09bf0850d10fef26a2/C of b7a5c1e2b432cf09bf0850d10fef26a2 into b3348110858645ed8e0bef1bcc49515a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:24,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:24,726 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., storeName=b7a5c1e2b432cf09bf0850d10fef26a2/C, priority=13, startTime=1733630363847; duration=0sec 2024-12-08T03:59:24,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:24,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7a5c1e2b432cf09bf0850d10fef26a2:C 2024-12-08T03:59:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:24,787 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/42360f1b35e04d409e5cf81d73a834dd 2024-12-08T03:59:24,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/274d6b2d89e947a49722c5b259912308 is 50, key is test_row_0/B:col10/1733630362203/Put/seqid=0 2024-12-08T03:59:24,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742040_1216 (size=12301) 2024-12-08T03:59:25,205 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/274d6b2d89e947a49722c5b259912308 2024-12-08T03:59:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/ccbe544694254f0f8725c51be402c987 is 50, key is test_row_0/C:col10/1733630362203/Put/seqid=0 2024-12-08T03:59:25,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742041_1217 (size=12301) 2024-12-08T03:59:25,622 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/ccbe544694254f0f8725c51be402c987 2024-12-08T03:59:25,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/42360f1b35e04d409e5cf81d73a834dd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/42360f1b35e04d409e5cf81d73a834dd 2024-12-08T03:59:25,637 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/42360f1b35e04d409e5cf81d73a834dd, entries=150, sequenceid=368, filesize=30.5 K 2024-12-08T03:59:25,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/274d6b2d89e947a49722c5b259912308 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/274d6b2d89e947a49722c5b259912308 2024-12-08T03:59:25,643 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/274d6b2d89e947a49722c5b259912308, entries=150, sequenceid=368, filesize=12.0 K 2024-12-08T03:59:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/ccbe544694254f0f8725c51be402c987 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/ccbe544694254f0f8725c51be402c987 2024-12-08T03:59:25,648 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/ccbe544694254f0f8725c51be402c987, entries=150, sequenceid=368, filesize=12.0 K 2024-12-08T03:59:25,649 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1686ms, sequenceid=368, compaction requested=false 2024-12-08T03:59:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for b7a5c1e2b432cf09bf0850d10fef26a2: 2024-12-08T03:59:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-08T03:59:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-08T03:59:25,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-08T03:59:25,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0700 sec 2024-12-08T03:59:25,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.0740 sec 2024-12-08T03:59:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-08T03:59:26,753 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:26,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:59:28,261 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-12-08T03:59:28,261 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-12-08T03:59:28,261 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:28,261 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:28,262 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,262 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,262 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,262 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:28,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:28,302 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-12-08T03:59:28,302 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:28,302 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,302 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:28,321 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-12-08T03:59:28,321 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:28,321 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,321 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4869 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4607 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2043 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6127 rows 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2037 2024-12-08T03:59:28,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6109 rows 2024-12-08T03:59:28,322 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:59:28,323 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:59:28,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:28,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:28,324 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:59:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:59:28,329 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630368329"}]},"ts":"1733630368329"} 2024-12-08T03:59:28,331 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:59:28,331 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:59:28,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:59:28,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, UNASSIGN}] 2024-12-08T03:59:28,335 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, UNASSIGN 2024-12-08T03:59:28,336 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:28,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, UNASSIGN because future has completed 2024-12-08T03:59:28,338 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:59:28,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:59:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:59:28,492 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:28,492 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:59:28,492 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing b7a5c1e2b432cf09bf0850d10fef26a2, disabling compactions & flushes 2024-12-08T03:59:28,492 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:28,492 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:28,493 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. after waiting 1 ms 2024-12-08T03:59:28,493 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:28,493 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing b7a5c1e2b432cf09bf0850d10fef26a2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T03:59:28,494 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=A 2024-12-08T03:59:28,494 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:28,494 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=B 2024-12-08T03:59:28,494 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:28,494 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7a5c1e2b432cf09bf0850d10fef26a2, store=C 2024-12-08T03:59:28,494 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:28,503 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208478a75ac3d6f49a19e1051c8eeb445e2_b7a5c1e2b432cf09bf0850d10fef26a2 is 50, key is test_row_0/A:col10/1733630368298/Put/seqid=0 2024-12-08T03:59:28,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742042_1218 (size=9914) 2024-12-08T03:59:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:59:28,908 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:28,918 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208478a75ac3d6f49a19e1051c8eeb445e2_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208478a75ac3d6f49a19e1051c8eeb445e2_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:28,919 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/b8c0dea5fe954a128f6d633bfdc8dd09, store: [table=TestAcidGuarantees family=A region=b7a5c1e2b432cf09bf0850d10fef26a2] 2024-12-08T03:59:28,919 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/b8c0dea5fe954a128f6d633bfdc8dd09 is 175, key is test_row_0/A:col10/1733630368298/Put/seqid=0 2024-12-08T03:59:28,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742043_1219 (size=22561) 2024-12-08T03:59:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:59:29,324 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/b8c0dea5fe954a128f6d633bfdc8dd09 2024-12-08T03:59:29,371 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/ea3aebe1c0804dfbb85db836e6252ff5 is 50, key is test_row_0/B:col10/1733630368298/Put/seqid=0 2024-12-08T03:59:29,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742044_1220 (size=9857) 2024-12-08T03:59:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:59:29,776 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/ea3aebe1c0804dfbb85db836e6252ff5 2024-12-08T03:59:29,787 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/922f96e03288455585f7da3e9304341b is 50, key is test_row_0/C:col10/1733630368298/Put/seqid=0 2024-12-08T03:59:29,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742045_1221 (size=9857) 2024-12-08T03:59:30,193 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/922f96e03288455585f7da3e9304341b 2024-12-08T03:59:30,204 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/A/b8c0dea5fe954a128f6d633bfdc8dd09 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/b8c0dea5fe954a128f6d633bfdc8dd09 2024-12-08T03:59:30,210 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/b8c0dea5fe954a128f6d633bfdc8dd09, entries=100, sequenceid=379, filesize=22.0 K 2024-12-08T03:59:30,211 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/B/ea3aebe1c0804dfbb85db836e6252ff5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/ea3aebe1c0804dfbb85db836e6252ff5 2024-12-08T03:59:30,215 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/ea3aebe1c0804dfbb85db836e6252ff5, entries=100, sequenceid=379, filesize=9.6 K 2024-12-08T03:59:30,216 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/.tmp/C/922f96e03288455585f7da3e9304341b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/922f96e03288455585f7da3e9304341b 2024-12-08T03:59:30,220 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/922f96e03288455585f7da3e9304341b, entries=100, sequenceid=379, filesize=9.6 K 2024-12-08T03:59:30,221 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1728ms, sequenceid=379, compaction requested=true 2024-12-08T03:59:30,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/bcab076596bb4ae2bc36ae19a4c6bce0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e35c7cc40f9c4196b573654c508c7f8f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e998e36484684bdeb426f714e98241be, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/27451ed5845743ac9e106bc761596d4d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/adc13df5f3934ab58bc432910b58545d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/694715b111c54e63aaf541ff9530a829, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9] to archive 2024-12-08T03:59:30,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:59:30,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/817ed00af7a342018cf4c3cc29490c4a 2024-12-08T03:59:30,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/41c8f89adfd74d7d89a6dae474f806bd 2024-12-08T03:59:30,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/bcab076596bb4ae2bc36ae19a4c6bce0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/bcab076596bb4ae2bc36ae19a4c6bce0 2024-12-08T03:59:30,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/aa8bd27238e24626b10525bfaecd8f07 2024-12-08T03:59:30,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/5317b9290bb344f2906e4b008ea2e5fb 2024-12-08T03:59:30,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/df5cac304eb54ec09172b3cbe8a64ee5 2024-12-08T03:59:30,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/01baa4fb16a449448ecf777ae001d94c 2024-12-08T03:59:30,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e35c7cc40f9c4196b573654c508c7f8f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e35c7cc40f9c4196b573654c508c7f8f 2024-12-08T03:59:30,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/cb8c1c4bb67648768063c5c42b4c877e 2024-12-08T03:59:30,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/79b12dd825cd4c3d935613b8f5e4c888 2024-12-08T03:59:30,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3c53a8f6176243948b6738edda90640e 2024-12-08T03:59:30,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e998e36484684bdeb426f714e98241be to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/e998e36484684bdeb426f714e98241be 2024-12-08T03:59:30,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f35a641b858848c994d1a29b0396ed01 2024-12-08T03:59:30,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/97105618f6ff4e5daf84db108aacdd31 2024-12-08T03:59:30,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/27451ed5845743ac9e106bc761596d4d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/27451ed5845743ac9e106bc761596d4d 2024-12-08T03:59:30,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/f699fbde7dd345d0a5b42e56b0212df1 2024-12-08T03:59:30,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/adc13df5f3934ab58bc432910b58545d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/adc13df5f3934ab58bc432910b58545d 2024-12-08T03:59:30,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/da80669f7a514c9b8da8245438d0e19a 2024-12-08T03:59:30,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/2a271e9e5e4d4fcb84c057cc06b57ce0 2024-12-08T03:59:30,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1913feef04164344b336dd1741064c4c 2024-12-08T03:59:30,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/694715b111c54e63aaf541ff9530a829 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/694715b111c54e63aaf541ff9530a829 2024-12-08T03:59:30,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/3ecf886a31c64023917b09fd5c9e0fc9 2024-12-08T03:59:30,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/7f8d44ff96e04b14bcacdd922deb1ac9 2024-12-08T03:59:30,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/039b06e5b82f4538b098c40e58bc0daa, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a81c1ee6eeef4340be7da87452e55b2b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/53c6c8e785804ac7be6002af6eed4c73, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/dcae451cf2c7440290474c6a5823a29b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/d33e887bbd354a7c984c5c61192f338c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/878a064367ba4f67900b8386789d6b79, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/00c69811a7464fdc881c5ff6e2540222, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/b8f6c317aa9c44e0aa8de275295e7357, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/35d6d03a7ab04de39ed0ee61916a075c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/78529a94da5c4c48a4e33044a9729744, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/5a08ddc5d8624114b21438a454ff7e96, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a458d12bd40e49d48fcd7c173ebc14eb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c4810715f3364873a2ed2245a982967a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/785a2d5c7b1d42daa66066adf8277fc4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c2dac7edac354c98aea386a2651712ae, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3d22a3b567a1472aab5a2c523037d8dd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/bf91287aeae04dd0b68e5681e33c021f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/2a92346571e54bf9b74b322e806cfc5c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/6dbcacc2b8c94d95b7ccfa2c89026278, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/84a45a31f17644399419aaaa44c12a28, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4a63fadb0a994af1af529bd1037310ae, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/28161b6ac15547d186b18d052fddf78d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3fd6bdd3706b42e8b7343d7f746406c4] to archive 2024-12-08T03:59:30,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:59:30,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/039b06e5b82f4538b098c40e58bc0daa to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/039b06e5b82f4538b098c40e58bc0daa 2024-12-08T03:59:30,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a81c1ee6eeef4340be7da87452e55b2b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a81c1ee6eeef4340be7da87452e55b2b 2024-12-08T03:59:30,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/53c6c8e785804ac7be6002af6eed4c73 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/53c6c8e785804ac7be6002af6eed4c73 2024-12-08T03:59:30,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/dcae451cf2c7440290474c6a5823a29b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/dcae451cf2c7440290474c6a5823a29b 2024-12-08T03:59:30,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/d33e887bbd354a7c984c5c61192f338c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/d33e887bbd354a7c984c5c61192f338c 2024-12-08T03:59:30,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/878a064367ba4f67900b8386789d6b79 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/878a064367ba4f67900b8386789d6b79 2024-12-08T03:59:30,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/00c69811a7464fdc881c5ff6e2540222 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/00c69811a7464fdc881c5ff6e2540222 2024-12-08T03:59:30,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/b8f6c317aa9c44e0aa8de275295e7357 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/b8f6c317aa9c44e0aa8de275295e7357 2024-12-08T03:59:30,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/35d6d03a7ab04de39ed0ee61916a075c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/35d6d03a7ab04de39ed0ee61916a075c 2024-12-08T03:59:30,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/78529a94da5c4c48a4e33044a9729744 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/78529a94da5c4c48a4e33044a9729744 2024-12-08T03:59:30,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/5a08ddc5d8624114b21438a454ff7e96 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/5a08ddc5d8624114b21438a454ff7e96 2024-12-08T03:59:30,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a458d12bd40e49d48fcd7c173ebc14eb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/a458d12bd40e49d48fcd7c173ebc14eb 2024-12-08T03:59:30,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c4810715f3364873a2ed2245a982967a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c4810715f3364873a2ed2245a982967a 2024-12-08T03:59:30,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/785a2d5c7b1d42daa66066adf8277fc4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/785a2d5c7b1d42daa66066adf8277fc4 2024-12-08T03:59:30,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c2dac7edac354c98aea386a2651712ae to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/c2dac7edac354c98aea386a2651712ae 2024-12-08T03:59:30,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3d22a3b567a1472aab5a2c523037d8dd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3d22a3b567a1472aab5a2c523037d8dd 2024-12-08T03:59:30,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/bf91287aeae04dd0b68e5681e33c021f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/bf91287aeae04dd0b68e5681e33c021f 2024-12-08T03:59:30,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/2a92346571e54bf9b74b322e806cfc5c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/2a92346571e54bf9b74b322e806cfc5c 2024-12-08T03:59:30,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/6dbcacc2b8c94d95b7ccfa2c89026278 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/6dbcacc2b8c94d95b7ccfa2c89026278 2024-12-08T03:59:30,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/84a45a31f17644399419aaaa44c12a28 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/84a45a31f17644399419aaaa44c12a28 2024-12-08T03:59:30,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4a63fadb0a994af1af529bd1037310ae to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4a63fadb0a994af1af529bd1037310ae 2024-12-08T03:59:30,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/28161b6ac15547d186b18d052fddf78d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/28161b6ac15547d186b18d052fddf78d 2024-12-08T03:59:30,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3fd6bdd3706b42e8b7343d7f746406c4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/3fd6bdd3706b42e8b7343d7f746406c4 2024-12-08T03:59:30,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/37e883716c1043fab58431c590389c57, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/630d58f412d6418d99cdcef1d4b48900, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/492457bb8049493fbdc8e1b5e20e2d15, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2778a321a982445ea200b6a89cbf703f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/a508799def664e90a00ba91b7915f952, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e35678e6daab497f9d4dff55417c4c0f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8d7b41c5af194788b047d53db1cefe9a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9c63d6a8654240c4a68beb4adc3f6e13, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/71d8538d28184f2aa9831316fd07e3f9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/c9dfa49351a849bd9d3e524ef4ab11e7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b463fc7ccbc040d2bc46255e83d5ce17, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/13a8b9be2e3641a79d35b2090fdfba1a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2f250eb750c44eeeb858869e1baeb761, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/cb29adc122fd41a49de7a5843672e7da, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/1d5e2a4fa32c4f77a5d35c74080e5f48, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9494d47b07d44532818389bfbf30d127, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/755636c7d5734f50a0f7215f11e26330, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/4702f2f4a15644ae8cb599de1ba29dbd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e6895f23947940889d1ebde79deadef2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b041be4faafa4ac5b8ab27535d3de62c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8288a653b87a48f8a4d7c5006bb3f62e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/3b0053c2f4804bc0acced259bdac66b8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/af9c83282f114b8aa63846e403fafd93] to archive 2024-12-08T03:59:30,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T03:59:30,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/37e883716c1043fab58431c590389c57 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/37e883716c1043fab58431c590389c57 2024-12-08T03:59:30,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/630d58f412d6418d99cdcef1d4b48900 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/630d58f412d6418d99cdcef1d4b48900 2024-12-08T03:59:30,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/492457bb8049493fbdc8e1b5e20e2d15 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/492457bb8049493fbdc8e1b5e20e2d15 2024-12-08T03:59:30,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2778a321a982445ea200b6a89cbf703f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2778a321a982445ea200b6a89cbf703f 2024-12-08T03:59:30,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/a508799def664e90a00ba91b7915f952 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/a508799def664e90a00ba91b7915f952 2024-12-08T03:59:30,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e35678e6daab497f9d4dff55417c4c0f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e35678e6daab497f9d4dff55417c4c0f 2024-12-08T03:59:30,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8d7b41c5af194788b047d53db1cefe9a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8d7b41c5af194788b047d53db1cefe9a 2024-12-08T03:59:30,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9c63d6a8654240c4a68beb4adc3f6e13 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9c63d6a8654240c4a68beb4adc3f6e13 2024-12-08T03:59:30,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/71d8538d28184f2aa9831316fd07e3f9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/71d8538d28184f2aa9831316fd07e3f9 2024-12-08T03:59:30,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/c9dfa49351a849bd9d3e524ef4ab11e7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/c9dfa49351a849bd9d3e524ef4ab11e7 2024-12-08T03:59:30,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b463fc7ccbc040d2bc46255e83d5ce17 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b463fc7ccbc040d2bc46255e83d5ce17 2024-12-08T03:59:30,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/13a8b9be2e3641a79d35b2090fdfba1a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/13a8b9be2e3641a79d35b2090fdfba1a 2024-12-08T03:59:30,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2f250eb750c44eeeb858869e1baeb761 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/2f250eb750c44eeeb858869e1baeb761 2024-12-08T03:59:30,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/cb29adc122fd41a49de7a5843672e7da to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/cb29adc122fd41a49de7a5843672e7da 2024-12-08T03:59:30,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/1d5e2a4fa32c4f77a5d35c74080e5f48 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/1d5e2a4fa32c4f77a5d35c74080e5f48 2024-12-08T03:59:30,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9494d47b07d44532818389bfbf30d127 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/9494d47b07d44532818389bfbf30d127 2024-12-08T03:59:30,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/755636c7d5734f50a0f7215f11e26330 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/755636c7d5734f50a0f7215f11e26330 2024-12-08T03:59:30,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/4702f2f4a15644ae8cb599de1ba29dbd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/4702f2f4a15644ae8cb599de1ba29dbd 2024-12-08T03:59:30,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e6895f23947940889d1ebde79deadef2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/e6895f23947940889d1ebde79deadef2 2024-12-08T03:59:30,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b041be4faafa4ac5b8ab27535d3de62c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b041be4faafa4ac5b8ab27535d3de62c 2024-12-08T03:59:30,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8288a653b87a48f8a4d7c5006bb3f62e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/8288a653b87a48f8a4d7c5006bb3f62e 2024-12-08T03:59:30,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/3b0053c2f4804bc0acced259bdac66b8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/3b0053c2f4804bc0acced259bdac66b8 2024-12-08T03:59:30,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/af9c83282f114b8aa63846e403fafd93 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/af9c83282f114b8aa63846e403fafd93 2024-12-08T03:59:30,303 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-12-08T03:59:30,304 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. 2024-12-08T03:59:30,304 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for b7a5c1e2b432cf09bf0850d10fef26a2: Waiting for close lock at 1733630368492Running coprocessor pre-close hooks at 1733630368492Disabling compacts and flushes for region at 1733630368492Disabling writes for close at 1733630368493 (+1 ms)Obtaining lock to block concurrent updates at 1733630368493Preparing flush snapshotting stores in b7a5c1e2b432cf09bf0850d10fef26a2 at 1733630368493Finished memstore snapshotting TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733630368494 (+1 ms)Flushing stores of TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2. at 1733630368496 (+2 ms)Flushing b7a5c1e2b432cf09bf0850d10fef26a2/A: creating writer at 1733630368496Flushing mob file b7a5c1e2b432cf09bf0850d10fef26a2/A: appending metadata at 1733630368502 (+6 ms)Flushing mob file b7a5c1e2b432cf09bf0850d10fef26a2/A: closing flushed file at 1733630368502Flushing b7a5c1e2b432cf09bf0850d10fef26a2/A: appending metadata at 1733630368919 (+417 ms)Flushing b7a5c1e2b432cf09bf0850d10fef26a2/A: closing flushed file at 1733630368919Flushing b7a5c1e2b432cf09bf0850d10fef26a2/B: creating writer at 1733630369364 (+445 ms)Flushing b7a5c1e2b432cf09bf0850d10fef26a2/B: appending metadata at 1733630369370 (+6 ms)Flushing b7a5c1e2b432cf09bf0850d10fef26a2/B: closing flushed file at 1733630369370Flushing b7a5c1e2b432cf09bf0850d10fef26a2/C: creating writer at 1733630369783 (+413 ms)Flushing b7a5c1e2b432cf09bf0850d10fef26a2/C: appending metadata at 1733630369787 (+4 ms)Flushing b7a5c1e2b432cf09bf0850d10fef26a2/C: closing flushed file at 1733630369787Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c3d0181: reopening flushed file at 1733630370203 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c752487: reopening flushed file at 1733630370210 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5275abf7: reopening flushed file at 1733630370215 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b7a5c1e2b432cf09bf0850d10fef26a2 in 1728ms, sequenceid=379, compaction requested=true at 1733630370221 (+6 ms)Writing region close event to WAL at 1733630370300 (+79 ms)Running coprocessor post-close hooks at 1733630370303 (+3 ms)Closed at 1733630370304 (+1 ms) 2024-12-08T03:59:30,305 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,306 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=b7a5c1e2b432cf09bf0850d10fef26a2, regionState=CLOSED 2024-12-08T03:59:30,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:59:30,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-08T03:59:30,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure b7a5c1e2b432cf09bf0850d10fef26a2, server=32df4b4bb439,39919,1733630308479 in 1.9700 sec 2024-12-08T03:59:30,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-08T03:59:30,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7a5c1e2b432cf09bf0850d10fef26a2, UNASSIGN in 1.9750 sec 2024-12-08T03:59:30,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-08T03:59:30,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9790 sec 2024-12-08T03:59:30,313 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630370312"}]},"ts":"1733630370312"} 2024-12-08T03:59:30,313 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T03:59:30,314 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T03:59:30,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9890 sec 2024-12-08T03:59:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T03:59:30,477 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:30,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T03:59:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,480 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:59:30,481 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,484 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,488 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/recovered.edits] 2024-12-08T03:59:30,494 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1b8efaf88131452d97397da3a627de0a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/1b8efaf88131452d97397da3a627de0a 2024-12-08T03:59:30,496 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/42360f1b35e04d409e5cf81d73a834dd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/42360f1b35e04d409e5cf81d73a834dd 2024-12-08T03:59:30,498 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/b8c0dea5fe954a128f6d633bfdc8dd09 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/A/b8c0dea5fe954a128f6d633bfdc8dd09 2024-12-08T03:59:30,503 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/274d6b2d89e947a49722c5b259912308 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/274d6b2d89e947a49722c5b259912308 2024-12-08T03:59:30,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4dfc0eb2745c4093a2d892c5cacbf89b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/4dfc0eb2745c4093a2d892c5cacbf89b 2024-12-08T03:59:30,507 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/ea3aebe1c0804dfbb85db836e6252ff5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/B/ea3aebe1c0804dfbb85db836e6252ff5 2024-12-08T03:59:30,511 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/922f96e03288455585f7da3e9304341b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/922f96e03288455585f7da3e9304341b 2024-12-08T03:59:30,512 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b3348110858645ed8e0bef1bcc49515a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/b3348110858645ed8e0bef1bcc49515a 2024-12-08T03:59:30,513 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/ccbe544694254f0f8725c51be402c987 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/C/ccbe544694254f0f8725c51be402c987 2024-12-08T03:59:30,515 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/recovered.edits/382.seqid to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2/recovered.edits/382.seqid 2024-12-08T03:59:30,516 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,516 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T03:59:30,516 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:59:30,517 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T03:59:30,520 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208052fb624509c45e2af152d0da49fe8ac_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208052fb624509c45e2af152d0da49fe8ac_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,521 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082710c669ddce4f7f8972487cb68ac8cd_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082710c669ddce4f7f8972487cb68ac8cd_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,522 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208478a75ac3d6f49a19e1051c8eeb445e2_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208478a75ac3d6f49a19e1051c8eeb445e2_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,523 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084baeb1fbba454f99a177b40c49d0c2a8_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084baeb1fbba454f99a177b40c49d0c2a8_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,524 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084f5f8764145a4533b5e82d79cdc59c17_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084f5f8764145a4533b5e82d79cdc59c17_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,525 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086b4f419803a24476950c617cc693c233_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086b4f419803a24476950c617cc693c233_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,527 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086f076bdaa5b8416aaf5f85b3a2b20ad3_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086f076bdaa5b8416aaf5f85b3a2b20ad3_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,528 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208845dcc0ef42146899d8e6203adadfcad_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208845dcc0ef42146899d8e6203adadfcad_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,529 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a02c1c338bab425990f9aa0de4aba352_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a02c1c338bab425990f9aa0de4aba352_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,531 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a426dcc322ae4d06b9b2eeb33716f6f2_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a426dcc322ae4d06b9b2eeb33716f6f2_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,533 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ab18f57956534806be6d8cfd1b407602_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ab18f57956534806be6d8cfd1b407602_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,534 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b2bd94d6de7d4d8780de9868c6ea3326_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b2bd94d6de7d4d8780de9868c6ea3326_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,535 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c0bdf4dfa99341959104a2de59f261ce_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c0bdf4dfa99341959104a2de59f261ce_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,536 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c3c4aeeba039422289f0a42e4ce33daa_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c3c4aeeba039422289f0a42e4ce33daa_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,537 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d4b28c4f7acd4a92a9224d5192b0a9f0_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d4b28c4f7acd4a92a9224d5192b0a9f0_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,538 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dcd0ac10c788485b85b62234ef85bdec_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dcd0ac10c788485b85b62234ef85bdec_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,539 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e9ef5566df9d458e9b0260d95bcc84fd_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e9ef5566df9d458e9b0260d95bcc84fd_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,540 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ec65e9c56c7243e9917c222e87334bbd_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ec65e9c56c7243e9917c222e87334bbd_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,541 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f9e478332eb6483a83e855ad1fcf543a_b7a5c1e2b432cf09bf0850d10fef26a2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f9e478332eb6483a83e855ad1fcf543a_b7a5c1e2b432cf09bf0850d10fef26a2 2024-12-08T03:59:30,541 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T03:59:30,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,546 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T03:59:30,547 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T03:59:30,548 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,548 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T03:59:30,549 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733630370548"}]},"ts":"9223372036854775807"} 2024-12-08T03:59:30,550 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T03:59:30,550 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b7a5c1e2b432cf09bf0850d10fef26a2, NAME => 'TestAcidGuarantees,,1733630340161.b7a5c1e2b432cf09bf0850d10fef26a2.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T03:59:30,550 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T03:59:30,550 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733630370550"}]},"ts":"9223372036854775807"} 2024-12-08T03:59:30,552 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T03:59:30,552 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-12-08T03:59:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T03:59:30,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T03:59:30,588 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:30,600 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 377), ProcessCount=11 (was 11), AvailableMemoryMB=16567 (was 16686) 2024-12-08T03:59:30,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=16567 2024-12-08T03:59:30,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T03:59:30,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T03:59:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:30,611 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T03:59:30,612 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:30,612 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-08T03:59:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T03:59:30,613 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T03:59:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742046_1222 (size=1061) 2024-12-08T03:59:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T03:59:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T03:59:31,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T03:59:31,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742047_1223 (size=53) 2024-12-08T03:59:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T03:59:31,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:59:31,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eaa9250e0e196d2f3604a9c3f7ea0f7e, disabling compactions & flushes 2024-12-08T03:59:31,434 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. after waiting 0 ms 2024-12-08T03:59:31,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,434 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,434 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: Waiting for close lock at 1733630371434Disabling compacts and flushes for region at 1733630371434Disabling writes for close at 1733630371434Writing region close event to WAL at 1733630371434Closed at 1733630371434 2024-12-08T03:59:31,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T03:59:31,435 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733630371435"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630371435"}]},"ts":"1733630371435"} 2024-12-08T03:59:31,437 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T03:59:31,438 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T03:59:31,438 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630371438"}]},"ts":"1733630371438"} 2024-12-08T03:59:31,440 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T03:59:31,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, ASSIGN}] 2024-12-08T03:59:31,441 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, ASSIGN 2024-12-08T03:59:31,442 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, ASSIGN; state=OFFLINE, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=false 2024-12-08T03:59:31,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=eaa9250e0e196d2f3604a9c3f7ea0f7e, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:31,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, ASSIGN because future has completed 2024-12-08T03:59:31,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:59:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T03:59:31,755 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,756 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} 2024-12-08T03:59:31,756 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,756 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T03:59:31,756 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,756 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,759 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,760 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:31,761 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa9250e0e196d2f3604a9c3f7ea0f7e columnFamilyName A 2024-12-08T03:59:31,761 DEBUG [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:31,761 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.HStore(327): Store=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:31,762 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,763 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:31,763 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa9250e0e196d2f3604a9c3f7ea0f7e columnFamilyName B 2024-12-08T03:59:31,763 DEBUG [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:31,764 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.HStore(327): Store=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:31,764 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,765 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T03:59:31,765 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaa9250e0e196d2f3604a9c3f7ea0f7e columnFamilyName C 2024-12-08T03:59:31,765 DEBUG [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T03:59:31,765 INFO [StoreOpener-eaa9250e0e196d2f3604a9c3f7ea0f7e-1 {}] regionserver.HStore(327): Store=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T03:59:31,765 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,765 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,766 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,766 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,767 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,767 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,768 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T03:59:31,769 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,771 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T03:59:31,771 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened eaa9250e0e196d2f3604a9c3f7ea0f7e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74461185, jitterRate=0.1095581203699112}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T03:59:31,771 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:31,772 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: Running coprocessor pre-open hook at 1733630371757Writing region info on filesystem at 1733630371757Initializing all the Stores at 1733630371758 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630371758Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630371758Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630371758Cleaning up temporary data from old regions at 1733630371767 (+9 ms)Running coprocessor post-open hooks at 1733630371771 (+4 ms)Region opened successfully at 1733630371772 (+1 ms) 2024-12-08T03:59:31,773 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., pid=65, masterSystemTime=1733630371750 2024-12-08T03:59:31,774 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,774 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:31,774 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=eaa9250e0e196d2f3604a9c3f7ea0f7e, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:31,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T03:59:31,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-08T03:59:31,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 in 179 msec 2024-12-08T03:59:31,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-08T03:59:31,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, ASSIGN in 337 msec 2024-12-08T03:59:31,779 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T03:59:31,779 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630371779"}]},"ts":"1733630371779"} 2024-12-08T03:59:31,780 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T03:59:31,781 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T03:59:31,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-08T03:59:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-08T03:59:32,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:32,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e031f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cddb7e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,776 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,780 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T03:59:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T03:59:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e4bebd9 2024-12-08T03:59:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T03:59:32,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T03:59:32,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,786 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dec18fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,789 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c6befb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f09c3d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7980d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,796 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33562a69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305a1a72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6167c51d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aadbdad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,817 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a4610b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e63e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,821 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1935ca04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,823 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e297cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,825 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72550330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3467c31a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688037bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,833 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcaeeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,835 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d16d776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce72cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T03:59:32,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T03:59:32,842 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T03:59:32,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T03:59:32,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T03:59:32,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cfe2553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T03:59:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T03:59:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:32,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T03:59:32,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5be04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T03:59:32,846 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,847 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,848 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,849 DEBUG [Thread-1069 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,850 DEBUG [Thread-1069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,851 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,852 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,854 DEBUG [Thread-1054 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,854 DEBUG [Thread-1054 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,860 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,862 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,863 DEBUG [Thread-1050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,863 DEBUG [Thread-1065 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-08T03:59:32,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:32,867 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,867 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:32,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:32,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:32,868 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:32,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,870 DEBUG [Thread-1063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,870 DEBUG [Thread-1061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T03:59:32,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T03:59:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T03:59:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T03:59:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T03:59:32,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5cfeb19e210e4fb08eb5333a4b0bc1bd is 50, key is test_row_0/A:col10/1733630372864/Put/seqid=0 2024-12-08T03:59:32,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733630372888, max timestamp=1733630372914Num uniques 859; 2024-12-08T03:59:32,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T03:59:32,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733630372888, max timestamp=1733630372914Num uniques 859; 2024-12-08T03:59:32,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T03:59:32,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733630372888, max timestamp=1733630372914Num uniques 859; 2024-12-08T03:59:32,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T03:59:32,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742048_1224 (size=12001) 2024-12-08T03:59:32,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630432939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:32,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630432942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630432943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:32,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630432943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:32,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630432945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:33,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630433057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630433057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630433057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630433058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630433057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:33,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630433267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630433267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630433267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630433268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630433277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5cfeb19e210e4fb08eb5333a4b0bc1bd 2024-12-08T03:59:33,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/d0cdcffd761f4b34b933666b84be938f is 50, key is test_row_0/B:col10/1733630372864/Put/seqid=0 2024-12-08T03:59:33,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742049_1225 (size=12001) 2024-12-08T03:59:33,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/d0cdcffd761f4b34b933666b84be938f 2024-12-08T03:59:33,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/d0cef19501ba4663b26f0abea2044df3 is 50, key is test_row_0/C:col10/1733630372864/Put/seqid=0 2024-12-08T03:59:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:33,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742050_1226 (size=12001) 2024-12-08T03:59:33,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/d0cef19501ba4663b26f0abea2044df3 2024-12-08T03:59:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5cfeb19e210e4fb08eb5333a4b0bc1bd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cfeb19e210e4fb08eb5333a4b0bc1bd 2024-12-08T03:59:33,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cfeb19e210e4fb08eb5333a4b0bc1bd, entries=150, sequenceid=11, filesize=11.7 K 2024-12-08T03:59:33,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/d0cdcffd761f4b34b933666b84be938f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d0cdcffd761f4b34b933666b84be938f 2024-12-08T03:59:33,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d0cdcffd761f4b34b933666b84be938f, entries=150, sequenceid=11, filesize=11.7 K 2024-12-08T03:59:33,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/d0cef19501ba4663b26f0abea2044df3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d0cef19501ba4663b26f0abea2044df3 2024-12-08T03:59:33,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630433577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630433578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630433579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630433579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d0cef19501ba4663b26f0abea2044df3, entries=150, sequenceid=11, filesize=11.7 K 2024-12-08T03:59:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 728ms, sequenceid=11, compaction requested=false 2024-12-08T03:59:33,595 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T03:59:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:33,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-08T03:59:33,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:33,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:33,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:33,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:33,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:33,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:33,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/c7d83fb8085a4541aef6e964d56a6f1c is 50, key is test_row_0/A:col10/1733630373601/Put/seqid=0 2024-12-08T03:59:33,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630433646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742051_1227 (size=14341) 2024-12-08T03:59:33,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/c7d83fb8085a4541aef6e964d56a6f1c 2024-12-08T03:59:33,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a31b202f7561420798efc4757a098d66 is 50, key is test_row_0/B:col10/1733630373601/Put/seqid=0 2024-12-08T03:59:33,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742052_1228 (size=12001) 2024-12-08T03:59:33,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a31b202f7561420798efc4757a098d66 2024-12-08T03:59:33,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c8fdbeaac3264828af763e1d44791f06 is 50, key is test_row_0/C:col10/1733630373601/Put/seqid=0 2024-12-08T03:59:33,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630433757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:33,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742053_1229 (size=12001) 2024-12-08T03:59:33,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:33,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:33,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:33,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:33,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630433967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:34,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630434097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:34,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630434097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630434097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:34,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:34,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:34,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630434107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:34,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c8fdbeaac3264828af763e1d44791f06 2024-12-08T03:59:34,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/c7d83fb8085a4541aef6e964d56a6f1c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c7d83fb8085a4541aef6e964d56a6f1c 2024-12-08T03:59:34,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c7d83fb8085a4541aef6e964d56a6f1c, entries=200, sequenceid=48, filesize=14.0 K 2024-12-08T03:59:34,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a31b202f7561420798efc4757a098d66 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a31b202f7561420798efc4757a098d66 2024-12-08T03:59:34,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a31b202f7561420798efc4757a098d66, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T03:59:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c8fdbeaac3264828af763e1d44791f06 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c8fdbeaac3264828af763e1d44791f06 2024-12-08T03:59:34,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c8fdbeaac3264828af763e1d44791f06, entries=150, sequenceid=48, filesize=11.7 K 2024-12-08T03:59:34,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 586ms, sequenceid=48, compaction requested=false 2024-12-08T03:59:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:34,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T03:59:34,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:34,258 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T03:59:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:34,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5452bcd61aa040a495242a6c57fe0778 is 50, key is test_row_0/A:col10/1733630373619/Put/seqid=0 2024-12-08T03:59:34,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742054_1230 (size=12001) 2024-12-08T03:59:34,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:34,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:34,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630374334, max timestamp=1733630374365Num uniques 859; 2024-12-08T03:59:34,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:34,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630374334, max timestamp=1733630374365Num uniques 859; 2024-12-08T03:59:34,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630374334, max timestamp=1733630374365Num uniques 859; 2024-12-08T03:59:34,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:34,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630374441, max timestamp=1733630374473Num uniques 1018; 2024-12-08T03:59:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630374441, max timestamp=1733630374473Num uniques 1018; 2024-12-08T03:59:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T03:59:34,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630374441, max timestamp=1733630374473Num uniques 1018; 2024-12-08T03:59:34,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T03:59:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T03:59:34,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733630374541, max timestamp=1733630374583Num uniques 1027; 2024-12-08T03:59:34,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733630374541, max timestamp=1733630374583Num uniques 1027; 2024-12-08T03:59:34,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T03:59:34,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T03:59:34,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733630374541, max timestamp=1733630374583Num uniques 1027; 2024-12-08T03:59:34,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-12-08T03:59:34,678 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5452bcd61aa040a495242a6c57fe0778 2024-12-08T03:59:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/62b1458f8377411ebd7e3fcdead98e7e is 50, key is test_row_0/B:col10/1733630373619/Put/seqid=0 2024-12-08T03:59:34,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742055_1231 (size=12001) 2024-12-08T03:59:34,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733630374673, max timestamp=1733630374694Num uniques 1036; 2024-12-08T03:59:34,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733630374673, max timestamp=1733630374694Num uniques 1036; 2024-12-08T03:59:34,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733630374673, max timestamp=1733630374694Num uniques 1009; 2024-12-08T03:59:34,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T03:59:34,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T03:59:34,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-12-08T03:59:34,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733630374745, max timestamp=1733630374775Num uniques 1045; 2024-12-08T03:59:34,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-08T03:59:34,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733630374745, max timestamp=1733630374775Num uniques 1045; 2024-12-08T03:59:34,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-08T03:59:34,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733630374745, max timestamp=1733630374775Num uniques 1045; 2024-12-08T03:59:34,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-08T03:59:34,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733630374824, max timestamp=1733630374872Num uniques 1054; 2024-12-08T03:59:34,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-08T03:59:34,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733630374824, max timestamp=1733630374872Num uniques 1054; 2024-12-08T03:59:34,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733630374824, max timestamp=1733630374872Num uniques 1054; 2024-12-08T03:59:34,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-08T03:59:34,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-08T03:59:34,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733630374929, max timestamp=1733630374964Num uniques 1013; 2024-12-08T03:59:34,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T03:59:34,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733630374929, max timestamp=1733630374964Num uniques 1013; 2024-12-08T03:59:34,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T03:59:34,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733630374929, max timestamp=1733630374964Num uniques 1013; 2024-12-08T03:59:34,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T03:59:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:35,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733630375022, max timestamp=1733630375065Num uniques 1022; 2024-12-08T03:59:35,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733630375022, max timestamp=1733630375065Num uniques 1022; 2024-12-08T03:59:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-08T03:59:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-08T03:59:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733630375022, max timestamp=1733630375065Num uniques 1022; 2024-12-08T03:59:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-08T03:59:35,093 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/62b1458f8377411ebd7e3fcdead98e7e 2024-12-08T03:59:35,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f13d93b114bd4856876d563d5cb9fbe5 is 50, key is test_row_0/C:col10/1733630373619/Put/seqid=0 2024-12-08T03:59:35,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742056_1232 (size=12001) 2024-12-08T03:59:35,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733630375116, max timestamp=1733630375120Num uniques 1031; 2024-12-08T03:59:35,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733630375116, max timestamp=1733630375120Num uniques 1031; 2024-12-08T03:59:35,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-08T03:59:35,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-08T03:59:35,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733630375116, max timestamp=1733630375120Num uniques 1031; 2024-12-08T03:59:35,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-08T03:59:35,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733630375127, max timestamp=1733630375139Num uniques 1040; 2024-12-08T03:59:35,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-08T03:59:35,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733630375127, max timestamp=1733630375139Num uniques 1040; 2024-12-08T03:59:35,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-08T03:59:35,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733630375127, max timestamp=1733630375139Num uniques 1040; 2024-12-08T03:59:35,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-08T03:59:35,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733630375155, max timestamp=1733630375161Num uniques 1049; 2024-12-08T03:59:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630435157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733630375155, max timestamp=1733630375161Num uniques 1049; 2024-12-08T03:59:35,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-12-08T03:59:35,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733630375155, max timestamp=1733630375161Num uniques 1049; 2024-12-08T03:59:35,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-12-08T03:59:35,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=13694, suffix heap size=137064,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=299 2024-12-08T03:59:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630435179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630435179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630435180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630435182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630435267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630435287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630435287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630435288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630435297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630435477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630435496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630435497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630435497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630435507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,512 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f13d93b114bd4856876d563d5cb9fbe5 2024-12-08T03:59:35,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5452bcd61aa040a495242a6c57fe0778 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5452bcd61aa040a495242a6c57fe0778 2024-12-08T03:59:35,522 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5452bcd61aa040a495242a6c57fe0778, entries=150, sequenceid=56, filesize=11.7 K 2024-12-08T03:59:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/62b1458f8377411ebd7e3fcdead98e7e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/62b1458f8377411ebd7e3fcdead98e7e 2024-12-08T03:59:35,526 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/62b1458f8377411ebd7e3fcdead98e7e, entries=150, sequenceid=56, filesize=11.7 K 2024-12-08T03:59:35,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f13d93b114bd4856876d563d5cb9fbe5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f13d93b114bd4856876d563d5cb9fbe5 2024-12-08T03:59:35,532 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f13d93b114bd4856876d563d5cb9fbe5, entries=150, sequenceid=56, filesize=11.7 K 2024-12-08T03:59:35,533 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 1275ms, sequenceid=56, compaction requested=true 2024-12-08T03:59:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:35,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-08T03:59:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-08T03:59:35,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-08T03:59:35,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6660 sec 2024-12-08T03:59:35,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6750 sec 2024-12-08T03:59:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:35,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=154.31 KB heapSize=431.63 KB 2024-12-08T03:59:35,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:35,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:35,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:35,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:35,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:35,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:35,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8c9dc8675fe444c08dbff070a276d427 is 50, key is test_row_0/A:col10/1733630375799/Put/seqid=0 2024-12-08T03:59:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742057_1233 (size=14691) 2024-12-08T03:59:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630435811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630435812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630435813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630435813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,820 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T03:59:35,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630435817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630435927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630435927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630435927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630435928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630436137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630436137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630436137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630436138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8c9dc8675fe444c08dbff070a276d427 2024-12-08T03:59:36,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0ab56cff08e84fa9963c383203af0850 is 50, key is test_row_0/B:col10/1733630375799/Put/seqid=0 2024-12-08T03:59:36,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742058_1234 (size=12301) 2024-12-08T03:59:36,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0ab56cff08e84fa9963c383203af0850 2024-12-08T03:59:36,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c80c1ce53ede447eb55358ac310d9164 is 50, key is test_row_0/C:col10/1733630375799/Put/seqid=0 2024-12-08T03:59:36,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742059_1235 (size=12301) 2024-12-08T03:59:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630436327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630436447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630436447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630436447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630436447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:36,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c80c1ce53ede447eb55358ac310d9164 2024-12-08T03:59:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8c9dc8675fe444c08dbff070a276d427 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8c9dc8675fe444c08dbff070a276d427 2024-12-08T03:59:36,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8c9dc8675fe444c08dbff070a276d427, entries=200, sequenceid=265, filesize=14.3 K 2024-12-08T03:59:36,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0ab56cff08e84fa9963c383203af0850 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0ab56cff08e84fa9963c383203af0850 2024-12-08T03:59:36,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0ab56cff08e84fa9963c383203af0850, entries=150, sequenceid=265, filesize=12.0 K 2024-12-08T03:59:36,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c80c1ce53ede447eb55358ac310d9164 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c80c1ce53ede447eb55358ac310d9164 2024-12-08T03:59:36,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c80c1ce53ede447eb55358ac310d9164, entries=150, sequenceid=265, filesize=12.0 K 2024-12-08T03:59:36,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.58 KB/441936, currentSize=33.54 KB/34350 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 898ms, sequenceid=265, compaction requested=true 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:36,698 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:36,698 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:36,700 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48304 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:36,701 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:36,701 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:36,701 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d0cdcffd761f4b34b933666b84be938f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a31b202f7561420798efc4757a098d66, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/62b1458f8377411ebd7e3fcdead98e7e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0ab56cff08e84fa9963c383203af0850] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=47.2 K 2024-12-08T03:59:36,701 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53034 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:36,701 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:36,701 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:36,701 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cfeb19e210e4fb08eb5333a4b0bc1bd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c7d83fb8085a4541aef6e964d56a6f1c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5452bcd61aa040a495242a6c57fe0778, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8c9dc8675fe444c08dbff070a276d427] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=51.8 K 2024-12-08T03:59:36,702 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cfeb19e210e4fb08eb5333a4b0bc1bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733630372858 2024-12-08T03:59:36,702 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cdcffd761f4b34b933666b84be938f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733630372858 2024-12-08T03:59:36,702 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d83fb8085a4541aef6e964d56a6f1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733630372940 2024-12-08T03:59:36,703 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a31b202f7561420798efc4757a098d66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733630372940 2024-12-08T03:59:36,703 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5452bcd61aa040a495242a6c57fe0778, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733630373619 2024-12-08T03:59:36,703 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b1458f8377411ebd7e3fcdead98e7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733630373619 2024-12-08T03:59:36,704 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9dc8675fe444c08dbff070a276d427, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733630375169 2024-12-08T03:59:36,704 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ab56cff08e84fa9963c383203af0850, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733630375178 2024-12-08T03:59:36,724 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:36,725 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/4da5e321c1d644fe8c9291f1e18c726d is 50, key is test_row_0/A:col10/1733630375799/Put/seqid=0 2024-12-08T03:59:36,729 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:36,730 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/fc45df01e9234c8bb2c2e52bb187fce3 is 50, key is test_row_0/B:col10/1733630375799/Put/seqid=0 2024-12-08T03:59:36,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742060_1236 (size=12439) 2024-12-08T03:59:36,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742061_1237 (size=12439) 2024-12-08T03:59:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:36,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:36,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:36,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:36,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:36,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:36,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:36,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:36,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8f00e7181eda4616b6269feb2ec3596f is 50, key is test_row_0/A:col10/1733630375810/Put/seqid=0 2024-12-08T03:59:36,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742062_1238 (size=17181) 2024-12-08T03:59:37,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630436980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733630376968, max timestamp=1733630376987Num uniques 859; 2024-12-08T03:59:37,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:37,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733630376968, max timestamp=1733630376987Num uniques 859; 2024-12-08T03:59:37,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T03:59:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733630376968, max timestamp=1733630376987Num uniques 859; 2024-12-08T03:59:37,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T03:59:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T03:59:37,038 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:37,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-08T03:59:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:59:37,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:37,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:37,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:37,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630437034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630437041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630437042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630437117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:59:37,149 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/4da5e321c1d644fe8c9291f1e18c726d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/4da5e321c1d644fe8c9291f1e18c726d 2024-12-08T03:59:37,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630437148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,157 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 4da5e321c1d644fe8c9291f1e18c726d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:37,157 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:37,157 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=12, startTime=1733630376698; duration=0sec 2024-12-08T03:59:37,157 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:37,157 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:37,157 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:37,158 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48304 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:37,158 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:37,158 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,158 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d0cef19501ba4663b26f0abea2044df3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c8fdbeaac3264828af763e1d44791f06, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f13d93b114bd4856876d563d5cb9fbe5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c80c1ce53ede447eb55358ac310d9164] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=47.2 K 2024-12-08T03:59:37,159 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0cef19501ba4663b26f0abea2044df3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733630372858 2024-12-08T03:59:37,159 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8fdbeaac3264828af763e1d44791f06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733630372940 2024-12-08T03:59:37,159 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f13d93b114bd4856876d563d5cb9fbe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733630373619 2024-12-08T03:59:37,159 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80c1ce53ede447eb55358ac310d9164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733630375178 2024-12-08T03:59:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630437157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630437157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,163 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/fc45df01e9234c8bb2c2e52bb187fce3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fc45df01e9234c8bb2c2e52bb187fce3 2024-12-08T03:59:37,169 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into fc45df01e9234c8bb2c2e52bb187fce3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:37,169 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:37,169 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=12, startTime=1733630376698; duration=0sec 2024-12-08T03:59:37,169 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:37,169 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:37,173 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:37,173 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/6a5862c5965d49f9a2a07281aa83b9b4 is 50, key is test_row_0/C:col10/1733630375799/Put/seqid=0 2024-12-08T03:59:37,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742063_1239 (size=12439) 2024-12-08T03:59:37,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:59:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630437327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630437337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,340 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:59:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:59:37,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630437357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630437367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8f00e7181eda4616b6269feb2ec3596f 2024-12-08T03:59:37,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630437367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/c3a47deb1d0d42f4ac733c150a0ee688 is 50, key is test_row_0/B:col10/1733630375810/Put/seqid=0 2024-12-08T03:59:37,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742064_1240 (size=12301) 2024-12-08T03:59:37,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:59:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,581 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/6a5862c5965d49f9a2a07281aa83b9b4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/6a5862c5965d49f9a2a07281aa83b9b4 2024-12-08T03:59:37,584 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into 6a5862c5965d49f9a2a07281aa83b9b4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:37,585 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:37,585 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=12, startTime=1733630376698; duration=0sec 2024-12-08T03:59:37,585 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:37,585 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:37,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630437637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:59:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:59:37,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630437667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630437677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630437677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:37,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/c3a47deb1d0d42f4ac733c150a0ee688 2024-12-08T03:59:37,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8358f061641947d9bb70e6cc848bd117 is 50, key is test_row_0/C:col10/1733630375810/Put/seqid=0 2024-12-08T03:59:37,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742065_1241 (size=12301) 2024-12-08T03:59:37,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8358f061641947d9bb70e6cc848bd117 2024-12-08T03:59:37,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8f00e7181eda4616b6269feb2ec3596f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8f00e7181eda4616b6269feb2ec3596f 2024-12-08T03:59:37,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8f00e7181eda4616b6269feb2ec3596f, entries=250, sequenceid=277, filesize=16.8 K 2024-12-08T03:59:37,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/c3a47deb1d0d42f4ac733c150a0ee688 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c3a47deb1d0d42f4ac733c150a0ee688 2024-12-08T03:59:37,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:59:37,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:37,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c3a47deb1d0d42f4ac733c150a0ee688, entries=150, sequenceid=277, filesize=12.0 K 2024-12-08T03:59:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:37,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8358f061641947d9bb70e6cc848bd117 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8358f061641947d9bb70e6cc848bd117 2024-12-08T03:59:37,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8358f061641947d9bb70e6cc848bd117, entries=150, sequenceid=277, filesize=12.0 K 2024-12-08T03:59:37,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=129.31 KB/132416 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 849ms, sequenceid=277, compaction requested=false 2024-12-08T03:59:37,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:37,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T03:59:37,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:37,956 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-12-08T03:59:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:37,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/259fa28bc9c443299226de12a78b7649 is 50, key is test_row_0/A:col10/1733630377031/Put/seqid=0 2024-12-08T03:59:37,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742066_1242 (size=12301) 2024-12-08T03:59:37,970 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/259fa28bc9c443299226de12a78b7649 2024-12-08T03:59:37,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8 is 50, key is test_row_0/B:col10/1733630377031/Put/seqid=0 2024-12-08T03:59:37,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742067_1243 (size=12301) 2024-12-08T03:59:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:38,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:59:38,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630438177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630438179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630438187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630438187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630438287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,386 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8 2024-12-08T03:59:38,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f343af42048d4fbd951a0dea4af44350 is 50, key is test_row_0/C:col10/1733630377031/Put/seqid=0 2024-12-08T03:59:38,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742068_1244 (size=12301) 2024-12-08T03:59:38,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630438498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630438807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:38,815 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f343af42048d4fbd951a0dea4af44350 2024-12-08T03:59:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/259fa28bc9c443299226de12a78b7649 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/259fa28bc9c443299226de12a78b7649 2024-12-08T03:59:38,827 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/259fa28bc9c443299226de12a78b7649, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T03:59:38,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8 2024-12-08T03:59:38,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T03:59:38,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f343af42048d4fbd951a0dea4af44350 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f343af42048d4fbd951a0dea4af44350 2024-12-08T03:59:38,836 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f343af42048d4fbd951a0dea4af44350, entries=150, sequenceid=315, filesize=12.0 K 2024-12-08T03:59:38,837 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132416, heapSize ~364.49 KB/373240, currentSize=53.67 KB/54960 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 881ms, sequenceid=315, compaction requested=true 2024-12-08T03:59:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-08T03:59:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-08T03:59:38,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-08T03:59:38,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-12-08T03:59:38,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8020 sec 2024-12-08T03:59:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T03:59:39,188 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:39,189 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-08T03:59:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:59:39,190 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:39,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:39,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:39,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T03:59:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:39,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7824a337bbc34b8390224c2f0b523675 is 50, key is test_row_0/A:col10/1733630378177/Put/seqid=0 2024-12-08T03:59:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742069_1245 (size=9857) 2024-12-08T03:59:39,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630439266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630439266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630439267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:59:39,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630439317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:39,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:39,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630439347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,349 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:39,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630439378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630439378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630439378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:59:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630439587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630439587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630439587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7824a337bbc34b8390224c2f0b523675 2024-12-08T03:59:39,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/3e08b1f825b241b4bfb330a86ee34ab5 is 50, key is test_row_0/B:col10/1733630378177/Put/seqid=0 2024-12-08T03:59:39,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742070_1246 (size=9857) 2024-12-08T03:59:39,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:59:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630439898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630439898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630439898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:39,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:39,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/3e08b1f825b241b4bfb330a86ee34ab5 2024-12-08T03:59:40,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2edef5ef05fa44e7800b31152ac762a1 is 50, key is test_row_0/C:col10/1733630378177/Put/seqid=0 2024-12-08T03:59:40,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742071_1247 (size=9857) 2024-12-08T03:59:40,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:40,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:40,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:59:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630440327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:40,332 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630440408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:40,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630440408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630440408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:40,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:40,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:40,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2edef5ef05fa44e7800b31152ac762a1 2024-12-08T03:59:40,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7824a337bbc34b8390224c2f0b523675 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7824a337bbc34b8390224c2f0b523675 2024-12-08T03:59:40,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7824a337bbc34b8390224c2f0b523675, entries=100, sequenceid=327, filesize=9.6 K 2024-12-08T03:59:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/3e08b1f825b241b4bfb330a86ee34ab5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3e08b1f825b241b4bfb330a86ee34ab5 2024-12-08T03:59:40,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3e08b1f825b241b4bfb330a86ee34ab5, entries=100, sequenceid=327, filesize=9.6 K 2024-12-08T03:59:40,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2edef5ef05fa44e7800b31152ac762a1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2edef5ef05fa44e7800b31152ac762a1 2024-12-08T03:59:40,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2edef5ef05fa44e7800b31152ac762a1, entries=100, sequenceid=327, filesize=9.6 K 2024-12-08T03:59:40,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 1277ms, sequenceid=327, compaction requested=true 2024-12-08T03:59:40,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:40,469 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:40,470 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:40,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:40,470 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:40,471 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:40,471 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,471 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/4da5e321c1d644fe8c9291f1e18c726d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8f00e7181eda4616b6269feb2ec3596f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/259fa28bc9c443299226de12a78b7649, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7824a337bbc34b8390224c2f0b523675] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=50.6 K 2024-12-08T03:59:40,471 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4da5e321c1d644fe8c9291f1e18c726d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733630375178 2024-12-08T03:59:40,472 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f00e7181eda4616b6269feb2ec3596f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733630375810 2024-12-08T03:59:40,472 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:40,472 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:40,472 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,472 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259fa28bc9c443299226de12a78b7649, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733630377031 2024-12-08T03:59:40,472 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fc45df01e9234c8bb2c2e52bb187fce3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c3a47deb1d0d42f4ac733c150a0ee688, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3e08b1f825b241b4bfb330a86ee34ab5] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=45.8 K 2024-12-08T03:59:40,473 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7824a337bbc34b8390224c2f0b523675, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733630378177 2024-12-08T03:59:40,473 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting fc45df01e9234c8bb2c2e52bb187fce3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733630375178 2024-12-08T03:59:40,474 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c3a47deb1d0d42f4ac733c150a0ee688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733630375810 2024-12-08T03:59:40,474 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bc7fa6c4df478b9cb73ecaa3fb45b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733630377031 2024-12-08T03:59:40,474 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e08b1f825b241b4bfb330a86ee34ab5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733630378177 2024-12-08T03:59:40,497 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:40,498 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/6ea485dbbbd549f6a94ae1064277d07f is 50, key is test_row_0/A:col10/1733630378177/Put/seqid=0 2024-12-08T03:59:40,509 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:40,510 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/c52b3b4e8d874bd08ab8bf717f832abd is 50, key is test_row_0/B:col10/1733630378177/Put/seqid=0 2024-12-08T03:59:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742072_1248 (size=12575) 2024-12-08T03:59:40,567 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/6ea485dbbbd549f6a94ae1064277d07f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6ea485dbbbd549f6a94ae1064277d07f 2024-12-08T03:59:40,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,574 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:40,579 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 6ea485dbbbd549f6a94ae1064277d07f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:40,579 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:40,580 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=12, startTime=1733630380469; duration=0sec 2024-12-08T03:59:40,580 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:40,580 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:40,580 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742073_1249 (size=12575) 2024-12-08T03:59:40,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/168c22b0cae94f41add12333717d8ea6 is 50, key is test_row_0/A:col10/1733630379217/Put/seqid=0 2024-12-08T03:59:40,585 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:40,585 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:40,585 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:40,585 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/6a5862c5965d49f9a2a07281aa83b9b4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8358f061641947d9bb70e6cc848bd117, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f343af42048d4fbd951a0dea4af44350, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2edef5ef05fa44e7800b31152ac762a1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=45.8 K 2024-12-08T03:59:40,586 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5862c5965d49f9a2a07281aa83b9b4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733630375178 2024-12-08T03:59:40,586 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8358f061641947d9bb70e6cc848bd117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733630375810 2024-12-08T03:59:40,587 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f343af42048d4fbd951a0dea4af44350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733630377031 2024-12-08T03:59:40,588 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2edef5ef05fa44e7800b31152ac762a1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733630378177 2024-12-08T03:59:40,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742074_1250 (size=12301) 2024-12-08T03:59:40,593 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/168c22b0cae94f41add12333717d8ea6 2024-12-08T03:59:40,602 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:40,603 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f0297f64ebc549509d370f8ca874febe is 50, key is test_row_0/C:col10/1733630378177/Put/seqid=0 2024-12-08T03:59:40,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0706ca4035fa462490c855a8f5f6418d is 50, key is test_row_0/B:col10/1733630379217/Put/seqid=0 2024-12-08T03:59:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742075_1251 (size=12575) 2024-12-08T03:59:40,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742076_1252 (size=12301) 2024-12-08T03:59:40,622 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0706ca4035fa462490c855a8f5f6418d 2024-12-08T03:59:40,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/db020012c46c4bff90a3e5db70c2d643 is 50, key is test_row_0/C:col10/1733630379217/Put/seqid=0 2024-12-08T03:59:40,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742077_1253 (size=12301) 2024-12-08T03:59:40,988 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/c52b3b4e8d874bd08ab8bf717f832abd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c52b3b4e8d874bd08ab8bf717f832abd 2024-12-08T03:59:40,993 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into c52b3b4e8d874bd08ab8bf717f832abd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:40,993 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:40,993 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=12, startTime=1733630380470; duration=0sec 2024-12-08T03:59:40,994 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:40,994 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:41,022 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/f0297f64ebc549509d370f8ca874febe as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f0297f64ebc549509d370f8ca874febe 2024-12-08T03:59:41,030 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into f0297f64ebc549509d370f8ca874febe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:41,031 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:41,031 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=12, startTime=1733630380470; duration=0sec 2024-12-08T03:59:41,031 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:41,032 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:41,074 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/db020012c46c4bff90a3e5db70c2d643 2024-12-08T03:59:41,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/168c22b0cae94f41add12333717d8ea6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/168c22b0cae94f41add12333717d8ea6 2024-12-08T03:59:41,086 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/168c22b0cae94f41add12333717d8ea6, entries=150, sequenceid=347, filesize=12.0 K 2024-12-08T03:59:41,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0706ca4035fa462490c855a8f5f6418d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0706ca4035fa462490c855a8f5f6418d 2024-12-08T03:59:41,096 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0706ca4035fa462490c855a8f5f6418d, entries=150, sequenceid=347, filesize=12.0 K 2024-12-08T03:59:41,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/db020012c46c4bff90a3e5db70c2d643 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/db020012c46c4bff90a3e5db70c2d643 2024-12-08T03:59:41,106 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/db020012c46c4bff90a3e5db70c2d643, entries=150, sequenceid=347, filesize=12.0 K 2024-12-08T03:59:41,108 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 534ms, sequenceid=347, compaction requested=false 2024-12-08T03:59:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-08T03:59:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-08T03:59:41,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-08T03:59:41,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-12-08T03:59:41,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9240 sec 2024-12-08T03:59:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T03:59:41,337 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:41,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-08T03:59:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:41,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:41,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:41,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:41,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T03:59:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:41,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/3d056b462af048c3880e947229abb3b8 is 50, key is test_row_0/A:col10/1733630381430/Put/seqid=0 2024-12-08T03:59:41,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742078_1254 (size=12301) 2024-12-08T03:59:41,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630381465, max timestamp=1733630381481Num uniques 859; 2024-12-08T03:59:41,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:41,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630381465, max timestamp=1733630381481Num uniques 859; 2024-12-08T03:59:41,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T03:59:41,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733630381472, max timestamp=1733630381481Num uniques 859; 2024-12-08T03:59:41,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T03:59:41,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630441516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630441516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630441517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630441627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630441628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630441628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:41,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630441837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630441837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630441837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:41,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/3d056b462af048c3880e947229abb3b8 2024-12-08T03:59:41,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/331afa8390624eaeae1ea851004f125c is 50, key is test_row_0/B:col10/1733630381430/Put/seqid=0 2024-12-08T03:59:41,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742079_1255 (size=12301) 2024-12-08T03:59:41,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:41,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:42,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630442147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630442147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630442147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:42,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/331afa8390624eaeae1ea851004f125c 2024-12-08T03:59:42,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/ed8b4dfa048246358ade725a483ece21 is 50, key is test_row_0/C:col10/1733630381430/Put/seqid=0 2024-12-08T03:59:42,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742080_1256 (size=12301) 2024-12-08T03:59:42,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/ed8b4dfa048246358ade725a483ece21 2024-12-08T03:59:42,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/3d056b462af048c3880e947229abb3b8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/3d056b462af048c3880e947229abb3b8 2024-12-08T03:59:42,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/3d056b462af048c3880e947229abb3b8, entries=150, sequenceid=361, filesize=12.0 K 2024-12-08T03:59:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/331afa8390624eaeae1ea851004f125c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/331afa8390624eaeae1ea851004f125c 2024-12-08T03:59:42,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/331afa8390624eaeae1ea851004f125c, entries=150, sequenceid=361, filesize=12.0 K 2024-12-08T03:59:42,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/ed8b4dfa048246358ade725a483ece21 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/ed8b4dfa048246358ade725a483ece21 2024-12-08T03:59:42,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/ed8b4dfa048246358ade725a483ece21, entries=150, sequenceid=361, filesize=12.0 K 2024-12-08T03:59:42,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.55 KB/134705 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 914ms, sequenceid=361, compaction requested=true 2024-12-08T03:59:42,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:42,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:42,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:42,350 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:42,350 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:42,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:42,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:42,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=138.26 KB heapSize=394.23 KB 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:42,351 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:42,351 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:42,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:42,351 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:42,351 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:42,351 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,351 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,352 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6ea485dbbbd549f6a94ae1064277d07f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/168c22b0cae94f41add12333717d8ea6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/3d056b462af048c3880e947229abb3b8] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.3 K 2024-12-08T03:59:42,352 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c52b3b4e8d874bd08ab8bf717f832abd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0706ca4035fa462490c855a8f5f6418d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/331afa8390624eaeae1ea851004f125c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.3 K 2024-12-08T03:59:42,352 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c52b3b4e8d874bd08ab8bf717f832abd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733630377040 2024-12-08T03:59:42,352 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea485dbbbd549f6a94ae1064277d07f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733630377040 2024-12-08T03:59:42,352 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0706ca4035fa462490c855a8f5f6418d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733630379217 2024-12-08T03:59:42,352 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 168c22b0cae94f41add12333717d8ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733630379217 2024-12-08T03:59:42,353 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 331afa8390624eaeae1ea851004f125c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630381430 2024-12-08T03:59:42,353 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d056b462af048c3880e947229abb3b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630381430 2024-12-08T03:59:42,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/45c6441bb6624600968be30584153a1f is 50, key is test_row_0/A:col10/1733630381515/Put/seqid=0 2024-12-08T03:59:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742081_1257 (size=14741) 2024-12-08T03:59:42,370 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:42,371 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b0890c3cb2954acf870b6d1c0b529d9c is 50, key is test_row_0/B:col10/1733630381430/Put/seqid=0 2024-12-08T03:59:42,376 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:42,377 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7429ca0d41f74a84b0172fd127c5515c is 50, key is test_row_0/A:col10/1733630381430/Put/seqid=0 2024-12-08T03:59:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742082_1258 (size=12677) 2024-12-08T03:59:42,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630442387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742083_1259 (size=12677) 2024-12-08T03:59:42,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,421 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7429ca0d41f74a84b0172fd127c5515c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7429ca0d41f74a84b0172fd127c5515c 2024-12-08T03:59:42,427 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 7429ca0d41f74a84b0172fd127c5515c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:42,427 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:42,427 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630382350; duration=0sec 2024-12-08T03:59:42,427 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:42,428 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:42,428 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:42,429 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:42,429 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:42,429 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,429 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f0297f64ebc549509d370f8ca874febe, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/db020012c46c4bff90a3e5db70c2d643, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/ed8b4dfa048246358ade725a483ece21] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.3 K 2024-12-08T03:59:42,429 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0297f64ebc549509d370f8ca874febe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733630377040 2024-12-08T03:59:42,430 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting db020012c46c4bff90a3e5db70c2d643, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733630379217 2024-12-08T03:59:42,430 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed8b4dfa048246358ade725a483ece21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630381430 2024-12-08T03:59:42,448 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:42,449 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/7c10692647e2406cac7fbe4dfa6ce87b is 50, key is test_row_0/C:col10/1733630381430/Put/seqid=0 2024-12-08T03:59:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742084_1260 (size=12677) 2024-12-08T03:59:42,473 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/7c10692647e2406cac7fbe4dfa6ce87b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/7c10692647e2406cac7fbe4dfa6ce87b 2024-12-08T03:59:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:42,479 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into 7c10692647e2406cac7fbe4dfa6ce87b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:42,479 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:42,479 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=13, startTime=1733630382351; duration=0sec 2024-12-08T03:59:42,480 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:42,480 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:42,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630442496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:42,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:42,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630442658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630442658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630442658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630442707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:42,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/45c6441bb6624600968be30584153a1f 2024-12-08T03:59:42,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/906072e2f0e14c8a816e3ed09f68eef2 is 50, key is test_row_0/B:col10/1733630381515/Put/seqid=0 2024-12-08T03:59:42,792 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b0890c3cb2954acf870b6d1c0b529d9c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b0890c3cb2954acf870b6d1c0b529d9c 2024-12-08T03:59:42,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742085_1261 (size=12301) 2024-12-08T03:59:42,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/906072e2f0e14c8a816e3ed09f68eef2 2024-12-08T03:59:42,798 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into b0890c3cb2954acf870b6d1c0b529d9c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:42,799 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:42,799 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630382350; duration=0sec 2024-12-08T03:59:42,799 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:42,799 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:42,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/fb128d9067344d278f65ee7a544fcab6 is 50, key is test_row_0/C:col10/1733630381515/Put/seqid=0 2024-12-08T03:59:42,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742086_1262 (size=12301) 2024-12-08T03:59:42,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:42,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:42,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630443017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:43,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:43,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/fb128d9067344d278f65ee7a544fcab6 2024-12-08T03:59:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/45c6441bb6624600968be30584153a1f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/45c6441bb6624600968be30584153a1f 2024-12-08T03:59:43,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/45c6441bb6624600968be30584153a1f, entries=200, sequenceid=398, filesize=14.4 K 2024-12-08T03:59:43,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/906072e2f0e14c8a816e3ed09f68eef2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/906072e2f0e14c8a816e3ed09f68eef2 2024-12-08T03:59:43,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:43,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/906072e2f0e14c8a816e3ed09f68eef2, entries=150, sequenceid=398, filesize=12.0 K 2024-12-08T03:59:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/fb128d9067344d278f65ee7a544fcab6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb128d9067344d278f65ee7a544fcab6 2024-12-08T03:59:43,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb128d9067344d278f65ee7a544fcab6, entries=150, sequenceid=398, filesize=12.0 K 2024-12-08T03:59:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-08T03:59:43,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.26 KB/141575, heapSize ~394.18 KB/403640, currentSize=40.25 KB/41220 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 1017ms, sequenceid=398, compaction requested=false 2024-12-08T03:59:43,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:43,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:43,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:43,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:43,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/36e9b570c4414d5b86b9615a8f0b17e6 is 50, key is test_row_1/A:col10/1733630382385/Put/seqid=0 2024-12-08T03:59:43,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742087_1263 (size=9857) 2024-12-08T03:59:43,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/36e9b570c4414d5b86b9615a8f0b17e6 2024-12-08T03:59:43,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/4b16f86142b84d9da17dcba4fccf4105 is 50, key is test_row_1/B:col10/1733630382385/Put/seqid=0 2024-12-08T03:59:43,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742088_1264 (size=9857) 2024-12-08T03:59:43,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630383430, max timestamp=1733630383449Num uniques 859; 2024-12-08T03:59:43,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630383430, max timestamp=1733630383449Num uniques 859; 2024-12-08T03:59:43,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:43,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:43,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630383430, max timestamp=1733630383449Num uniques 859; 2024-12-08T03:59:43,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:43,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:43,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630443515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630443527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630443627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:43,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630443667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630443667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,669 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:43,669 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630443677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,678 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:43,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/4b16f86142b84d9da17dcba4fccf4105 2024-12-08T03:59:43,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/17254057569b442d86108592d2a3aadf is 50, key is test_row_1/C:col10/1733630382385/Put/seqid=0 2024-12-08T03:59:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742089_1265 (size=9857) 2024-12-08T03:59:43,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/17254057569b442d86108592d2a3aadf 2024-12-08T03:59:43,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/36e9b570c4414d5b86b9615a8f0b17e6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/36e9b570c4414d5b86b9615a8f0b17e6 2024-12-08T03:59:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630443837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:43,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/36e9b570c4414d5b86b9615a8f0b17e6, entries=100, sequenceid=411, filesize=9.6 K 2024-12-08T03:59:43,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/4b16f86142b84d9da17dcba4fccf4105 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/4b16f86142b84d9da17dcba4fccf4105 2024-12-08T03:59:43,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/4b16f86142b84d9da17dcba4fccf4105, entries=100, sequenceid=411, filesize=9.6 K 2024-12-08T03:59:43,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/17254057569b442d86108592d2a3aadf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/17254057569b442d86108592d2a3aadf 2024-12-08T03:59:43,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/17254057569b442d86108592d2a3aadf, entries=100, sequenceid=411, filesize=9.6 K 2024-12-08T03:59:43,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 467ms, sequenceid=411, compaction requested=true 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:43,856 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:43,856 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:43,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:43,857 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37275 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:43,857 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:43,857 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:43,857 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:43,857 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,857 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,857 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b0890c3cb2954acf870b6d1c0b529d9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/906072e2f0e14c8a816e3ed09f68eef2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/4b16f86142b84d9da17dcba4fccf4105] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=34.0 K 2024-12-08T03:59:43,857 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7429ca0d41f74a84b0172fd127c5515c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/45c6441bb6624600968be30584153a1f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/36e9b570c4414d5b86b9615a8f0b17e6] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.4 K 2024-12-08T03:59:43,858 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b0890c3cb2954acf870b6d1c0b529d9c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630381430 2024-12-08T03:59:43,858 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7429ca0d41f74a84b0172fd127c5515c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630381430 2024-12-08T03:59:43,858 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 906072e2f0e14c8a816e3ed09f68eef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733630381513 2024-12-08T03:59:43,858 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c6441bb6624600968be30584153a1f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733630381513 2024-12-08T03:59:43,858 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b16f86142b84d9da17dcba4fccf4105, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733630382385 2024-12-08T03:59:43,859 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e9b570c4414d5b86b9615a8f0b17e6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733630382385 2024-12-08T03:59:43,867 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:43,867 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:43,868 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/3425a22c726c46be8418bb2c294a59d2 is 50, key is test_row_0/B:col10/1733630381515/Put/seqid=0 2024-12-08T03:59:43,868 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/e3485ea8a9c84b0282bfb1a3a2116aaf is 50, key is test_row_0/A:col10/1733630381515/Put/seqid=0 2024-12-08T03:59:43,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742091_1267 (size=12779) 2024-12-08T03:59:43,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742090_1266 (size=12779) 2024-12-08T03:59:43,903 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/e3485ea8a9c84b0282bfb1a3a2116aaf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e3485ea8a9c84b0282bfb1a3a2116aaf 2024-12-08T03:59:43,908 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into e3485ea8a9c84b0282bfb1a3a2116aaf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:43,908 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:43,908 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630383856; duration=0sec 2024-12-08T03:59:43,908 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:43,908 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:43,908 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:43,909 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:43,909 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:43,909 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,909 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/7c10692647e2406cac7fbe4dfa6ce87b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb128d9067344d278f65ee7a544fcab6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/17254057569b442d86108592d2a3aadf] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=34.0 K 2024-12-08T03:59:43,909 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c10692647e2406cac7fbe4dfa6ce87b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733630381430 2024-12-08T03:59:43,910 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb128d9067344d278f65ee7a544fcab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1733630381513 2024-12-08T03:59:43,910 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17254057569b442d86108592d2a3aadf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733630382385 2024-12-08T03:59:43,915 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#224 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:43,916 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/3b63367dec8d4f1994c8437109b7454f is 50, key is test_row_0/C:col10/1733630381515/Put/seqid=0 2024-12-08T03:59:43,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742092_1268 (size=12779) 2024-12-08T03:59:43,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T03:59:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:43,962 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T03:59:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:43,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/2f40718aa0674d3e84f60154d2e9e340 is 50, key is test_row_0/A:col10/1733630383509/Put/seqid=0 2024-12-08T03:59:43,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742093_1269 (size=12301) 2024-12-08T03:59:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:44,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:44,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630444179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:44,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630444287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:44,307 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/3425a22c726c46be8418bb2c294a59d2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3425a22c726c46be8418bb2c294a59d2 2024-12-08T03:59:44,314 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into 3425a22c726c46be8418bb2c294a59d2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:44,314 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:44,314 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630383856; duration=0sec 2024-12-08T03:59:44,314 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:44,314 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:44,326 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/3b63367dec8d4f1994c8437109b7454f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/3b63367dec8d4f1994c8437109b7454f 2024-12-08T03:59:44,333 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into 3b63367dec8d4f1994c8437109b7454f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:44,333 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:44,333 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=13, startTime=1733630383856; duration=0sec 2024-12-08T03:59:44,334 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:44,334 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:44,372 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/2f40718aa0674d3e84f60154d2e9e340 2024-12-08T03:59:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/f0fc07940c7e48d29774c2e33e2ca97e is 50, key is test_row_0/B:col10/1733630383509/Put/seqid=0 2024-12-08T03:59:44,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742094_1270 (size=12301) 2024-12-08T03:59:44,408 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/f0fc07940c7e48d29774c2e33e2ca97e 2024-12-08T03:59:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/d8e3436c9a6e46d0a5f0117111362c56 is 50, key is test_row_0/C:col10/1733630383509/Put/seqid=0 2024-12-08T03:59:44,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742095_1271 (size=12301) 2024-12-08T03:59:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630444497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:44,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630444537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:44,540 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:44,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630444807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:44,823 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/d8e3436c9a6e46d0a5f0117111362c56 2024-12-08T03:59:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/2f40718aa0674d3e84f60154d2e9e340 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/2f40718aa0674d3e84f60154d2e9e340 2024-12-08T03:59:44,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/2f40718aa0674d3e84f60154d2e9e340, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T03:59:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/f0fc07940c7e48d29774c2e33e2ca97e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/f0fc07940c7e48d29774c2e33e2ca97e 2024-12-08T03:59:44,838 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/f0fc07940c7e48d29774c2e33e2ca97e, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T03:59:44,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/d8e3436c9a6e46d0a5f0117111362c56 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d8e3436c9a6e46d0a5f0117111362c56 2024-12-08T03:59:44,844 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d8e3436c9a6e46d0a5f0117111362c56, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T03:59:44,846 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 884ms, sequenceid=449, compaction requested=false 2024-12-08T03:59:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-08T03:59:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-08T03:59:44,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-08T03:59:44,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5080 sec 2024-12-08T03:59:44,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 3.5120 sec 2024-12-08T03:59:45,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T03:59:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:45,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/90ff65e4b9e04ba480d97ad680053888 is 50, key is test_row_0/A:col10/1733630384170/Put/seqid=0 2024-12-08T03:59:45,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742096_1272 (size=12301) 2024-12-08T03:59:45,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/90ff65e4b9e04ba480d97ad680053888 2024-12-08T03:59:45,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6f1b939d9e2341dbba654b35e38c25f2 is 50, key is test_row_0/B:col10/1733630384170/Put/seqid=0 2024-12-08T03:59:45,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742097_1273 (size=12301) 2024-12-08T03:59:45,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630385375, max timestamp=1733630385403Num uniques 859; 2024-12-08T03:59:45,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630385375, max timestamp=1733630385403Num uniques 859; 2024-12-08T03:59:45,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:45,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:45,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630385375, max timestamp=1733630385403Num uniques 859; 2024-12-08T03:59:45,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630445484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T03:59:45,507 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:45,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-08T03:59:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:59:45,515 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:45,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:45,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630445597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:59:45,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T03:59:45,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:45,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:45,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:45,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630445677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:45,681 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:45,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630445687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:45,690 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:45,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630445698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:45,702 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:45,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:45,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6f1b939d9e2341dbba654b35e38c25f2 2024-12-08T03:59:45,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2cc73d0cf53545ac94de0c55dacdb4b3 is 50, key is test_row_0/C:col10/1733630384170/Put/seqid=0 2024-12-08T03:59:45,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630445807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:45,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742098_1274 (size=12301) 2024-12-08T03:59:45,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T03:59:45,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:45,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:45,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:45,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:59:45,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T03:59:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:45,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630446117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:46,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T03:59:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:59:46,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2cc73d0cf53545ac94de0c55dacdb4b3 2024-12-08T03:59:46,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/90ff65e4b9e04ba480d97ad680053888 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/90ff65e4b9e04ba480d97ad680053888 2024-12-08T03:59:46,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/90ff65e4b9e04ba480d97ad680053888, entries=150, sequenceid=462, filesize=12.0 K 2024-12-08T03:59:46,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6f1b939d9e2341dbba654b35e38c25f2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6f1b939d9e2341dbba654b35e38c25f2 2024-12-08T03:59:46,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6f1b939d9e2341dbba654b35e38c25f2, entries=150, sequenceid=462, filesize=12.0 K 2024-12-08T03:59:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2cc73d0cf53545ac94de0c55dacdb4b3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2cc73d0cf53545ac94de0c55dacdb4b3 2024-12-08T03:59:46,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2cc73d0cf53545ac94de0c55dacdb4b3, entries=150, sequenceid=462, filesize=12.0 K 2024-12-08T03:59:46,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 914ms, sequenceid=462, compaction requested=true 2024-12-08T03:59:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:46,233 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:46,233 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:46,235 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:46,235 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:46,235 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:46,235 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:46,235 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,235 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,235 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e3485ea8a9c84b0282bfb1a3a2116aaf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/2f40718aa0674d3e84f60154d2e9e340, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/90ff65e4b9e04ba480d97ad680053888] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.5 K 2024-12-08T03:59:46,235 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3425a22c726c46be8418bb2c294a59d2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/f0fc07940c7e48d29774c2e33e2ca97e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6f1b939d9e2341dbba654b35e38c25f2] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.5 K 2024-12-08T03:59:46,235 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3485ea8a9c84b0282bfb1a3a2116aaf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733630381515 2024-12-08T03:59:46,236 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3425a22c726c46be8418bb2c294a59d2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733630381515 2024-12-08T03:59:46,236 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f0fc07940c7e48d29774c2e33e2ca97e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733630383504 2024-12-08T03:59:46,236 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f40718aa0674d3e84f60154d2e9e340, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733630383504 2024-12-08T03:59:46,236 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90ff65e4b9e04ba480d97ad680053888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733630384163 2024-12-08T03:59:46,236 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1b939d9e2341dbba654b35e38c25f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733630384163 2024-12-08T03:59:46,246 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:46,247 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/e91671ded731486e9dac252357a18b79 is 50, key is test_row_0/A:col10/1733630384170/Put/seqid=0 2024-12-08T03:59:46,248 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:46,248 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/efa0d3f784324790b48afef626a92b2e is 50, key is test_row_0/B:col10/1733630384170/Put/seqid=0 2024-12-08T03:59:46,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742099_1275 (size=12881) 2024-12-08T03:59:46,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742100_1276 (size=12881) 2024-12-08T03:59:46,262 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/e91671ded731486e9dac252357a18b79 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e91671ded731486e9dac252357a18b79 2024-12-08T03:59:46,267 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/efa0d3f784324790b48afef626a92b2e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/efa0d3f784324790b48afef626a92b2e 2024-12-08T03:59:46,271 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into e91671ded731486e9dac252357a18b79(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:46,271 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:46,271 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630386233; duration=0sec 2024-12-08T03:59:46,271 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:46,271 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:46,271 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:46,275 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into efa0d3f784324790b48afef626a92b2e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:46,275 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:46,275 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630386233; duration=0sec 2024-12-08T03:59:46,275 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:46,275 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:46,275 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:46,275 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:46,275 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,275 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/3b63367dec8d4f1994c8437109b7454f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d8e3436c9a6e46d0a5f0117111362c56, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2cc73d0cf53545ac94de0c55dacdb4b3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.5 K 2024-12-08T03:59:46,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b63367dec8d4f1994c8437109b7454f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733630381515 2024-12-08T03:59:46,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e3436c9a6e46d0a5f0117111362c56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733630383504 2024-12-08T03:59:46,277 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cc73d0cf53545ac94de0c55dacdb4b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733630384163 2024-12-08T03:59:46,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,285 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:46,286 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:46,286 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/e98a915cb84b4a7bb3432fe8fad35f1d is 50, key is test_row_0/C:col10/1733630384170/Put/seqid=0 2024-12-08T03:59:46,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/c193534bfa8f451fb9c7be86d718bbb7 is 50, key is test_row_0/A:col10/1733630385469/Put/seqid=0 2024-12-08T03:59:46,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742101_1277 (size=12881) 2024-12-08T03:59:46,334 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/e98a915cb84b4a7bb3432fe8fad35f1d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/e98a915cb84b4a7bb3432fe8fad35f1d 2024-12-08T03:59:46,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742102_1278 (size=12301) 2024-12-08T03:59:46,337 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/c193534bfa8f451fb9c7be86d718bbb7 2024-12-08T03:59:46,344 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into e98a915cb84b4a7bb3432fe8fad35f1d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:46,344 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:46,344 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=13, startTime=1733630386233; duration=0sec 2024-12-08T03:59:46,344 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:46,344 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:46,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/453ccb9f957b4542afcd13c5013c9a21 is 50, key is test_row_0/B:col10/1733630385469/Put/seqid=0 2024-12-08T03:59:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742103_1279 (size=12301) 2024-12-08T03:59:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630446590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:46,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630446627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:59:46,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630446697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:46,753 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/453ccb9f957b4542afcd13c5013c9a21 2024-12-08T03:59:46,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/00af385ff45c4586bdfb2907f72b6d61 is 50, key is test_row_0/C:col10/1733630385469/Put/seqid=0 2024-12-08T03:59:46,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742104_1280 (size=12301) 2024-12-08T03:59:46,772 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/00af385ff45c4586bdfb2907f72b6d61 2024-12-08T03:59:46,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/c193534bfa8f451fb9c7be86d718bbb7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c193534bfa8f451fb9c7be86d718bbb7 2024-12-08T03:59:46,783 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c193534bfa8f451fb9c7be86d718bbb7, entries=150, sequenceid=500, filesize=12.0 K 2024-12-08T03:59:46,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/453ccb9f957b4542afcd13c5013c9a21 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/453ccb9f957b4542afcd13c5013c9a21 2024-12-08T03:59:46,788 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/453ccb9f957b4542afcd13c5013c9a21, entries=150, sequenceid=500, filesize=12.0 K 2024-12-08T03:59:46,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/00af385ff45c4586bdfb2907f72b6d61 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/00af385ff45c4586bdfb2907f72b6d61 2024-12-08T03:59:46,793 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/00af385ff45c4586bdfb2907f72b6d61, entries=150, sequenceid=500, filesize=12.0 K 2024-12-08T03:59:46,795 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 509ms, sequenceid=500, compaction requested=false 2024-12-08T03:59:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-08T03:59:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-08T03:59:46,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-08T03:59:46,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2800 sec 2024-12-08T03:59:46,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.2910 sec 2024-12-08T03:59:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:46,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T03:59:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:46,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/fb51373d57944999b0fe733524c1eb3a is 50, key is test_row_1/A:col10/1733630386588/Put/seqid=0 2024-12-08T03:59:46,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742105_1281 (size=12297) 2024-12-08T03:59:46,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630446979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:47,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630447087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:47,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630447297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:47,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/fb51373d57944999b0fe733524c1eb3a 2024-12-08T03:59:47,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/8d70d3aa62114557b67ee7e4aeee4235 is 50, key is test_row_1/B:col10/1733630386588/Put/seqid=0 2024-12-08T03:59:47,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742106_1282 (size=9857) 2024-12-08T03:59:47,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630447608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:47,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630447636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:47,638 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T03:59:47,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:47,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-08T03:59:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:59:47,659 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:47,660 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:47,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:59:47,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/8d70d3aa62114557b67ee7e4aeee4235 2024-12-08T03:59:47,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:59:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:47,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:47,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/02c2559cd0024ec3849a79424a638e13 is 50, key is test_row_1/C:col10/1733630386588/Put/seqid=0 2024-12-08T03:59:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742107_1283 (size=9857) 2024-12-08T03:59:47,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:59:47,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:47,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:47,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:59:48,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630448117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:48,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:59:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:48,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:48,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:48,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/02c2559cd0024ec3849a79424a638e13 2024-12-08T03:59:48,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/fb51373d57944999b0fe733524c1eb3a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/fb51373d57944999b0fe733524c1eb3a 2024-12-08T03:59:48,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/fb51373d57944999b0fe733524c1eb3a, entries=150, sequenceid=513, filesize=12.0 K 2024-12-08T03:59:48,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/8d70d3aa62114557b67ee7e4aeee4235 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8d70d3aa62114557b67ee7e4aeee4235 2024-12-08T03:59:48,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8d70d3aa62114557b67ee7e4aeee4235, entries=100, sequenceid=513, filesize=9.6 K 2024-12-08T03:59:48,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/02c2559cd0024ec3849a79424a638e13 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/02c2559cd0024ec3849a79424a638e13 2024-12-08T03:59:48,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/02c2559cd0024ec3849a79424a638e13, entries=100, sequenceid=513, filesize=9.6 K 2024-12-08T03:59:48,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 1354ms, sequenceid=513, compaction requested=true 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:48,263 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:48,263 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:48,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37479 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:48,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:48,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:48,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:48,272 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,272 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,272 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/efa0d3f784324790b48afef626a92b2e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/453ccb9f957b4542afcd13c5013c9a21, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8d70d3aa62114557b67ee7e4aeee4235] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=34.2 K 2024-12-08T03:59:48,272 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e91671ded731486e9dac252357a18b79, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c193534bfa8f451fb9c7be86d718bbb7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/fb51373d57944999b0fe733524c1eb3a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.6 K 2024-12-08T03:59:48,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91671ded731486e9dac252357a18b79, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733630384163 2024-12-08T03:59:48,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting efa0d3f784324790b48afef626a92b2e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733630384163 2024-12-08T03:59:48,273 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 453ccb9f957b4542afcd13c5013c9a21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733630385453 2024-12-08T03:59:48,273 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c193534bfa8f451fb9c7be86d718bbb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733630385453 2024-12-08T03:59:48,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d70d3aa62114557b67ee7e4aeee4235, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733630386588 2024-12-08T03:59:48,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb51373d57944999b0fe733524c1eb3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733630386584 2024-12-08T03:59:48,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T03:59:48,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,275 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T03:59:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:59:48,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/220e548a4d0f47f7aea2cede713db88c is 50, key is test_row_0/A:col10/1733630386971/Put/seqid=0 2024-12-08T03:59:48,289 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:48,290 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5cbe6ed1777a4edda9b01d874d568046 is 50, key is test_row_0/A:col10/1733630385469/Put/seqid=0 2024-12-08T03:59:48,290 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:48,290 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/fb51907d5df74cb7989faaa585ab8c36 is 50, key is test_row_0/B:col10/1733630385469/Put/seqid=0 2024-12-08T03:59:48,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742110_1286 (size=12983) 2024-12-08T03:59:48,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742108_1284 (size=12983) 2024-12-08T03:59:48,302 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/fb51907d5df74cb7989faaa585ab8c36 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fb51907d5df74cb7989faaa585ab8c36 2024-12-08T03:59:48,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742109_1285 (size=12301) 2024-12-08T03:59:48,304 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/220e548a4d0f47f7aea2cede713db88c 2024-12-08T03:59:48,352 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into fb51907d5df74cb7989faaa585ab8c36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:48,352 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:48,352 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630388263; duration=0sec 2024-12-08T03:59:48,352 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:48,352 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:48,352 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:48,353 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:48,353 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:48,353 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,353 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/e98a915cb84b4a7bb3432fe8fad35f1d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/00af385ff45c4586bdfb2907f72b6d61, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/02c2559cd0024ec3849a79424a638e13] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=34.2 K 2024-12-08T03:59:48,353 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e98a915cb84b4a7bb3432fe8fad35f1d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733630384163 2024-12-08T03:59:48,354 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 00af385ff45c4586bdfb2907f72b6d61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1733630385453 2024-12-08T03:59:48,354 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c2559cd0024ec3849a79424a638e13, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733630386588 2024-12-08T03:59:48,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a15f06783ece426faf3e381f237d859e is 50, key is test_row_0/B:col10/1733630386971/Put/seqid=0 2024-12-08T03:59:48,364 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:48,364 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/a4b8715d4ec34198bb656f5a171e4632 is 50, key is test_row_0/C:col10/1733630385469/Put/seqid=0 2024-12-08T03:59:48,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742111_1287 (size=12301) 2024-12-08T03:59:48,370 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a15f06783ece426faf3e381f237d859e 2024-12-08T03:59:48,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742112_1288 (size=12983) 2024-12-08T03:59:48,381 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/a4b8715d4ec34198bb656f5a171e4632 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/a4b8715d4ec34198bb656f5a171e4632 2024-12-08T03:59:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2d851ccf2b7148c898d356512ef66628 is 50, key is test_row_0/C:col10/1733630386971/Put/seqid=0 2024-12-08T03:59:48,388 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into a4b8715d4ec34198bb656f5a171e4632(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:48,388 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:48,388 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=13, startTime=1733630388263; duration=0sec 2024-12-08T03:59:48,388 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:48,388 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:48,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742113_1289 (size=12301) 2024-12-08T03:59:48,398 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2d851ccf2b7148c898d356512ef66628 2024-12-08T03:59:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/220e548a4d0f47f7aea2cede713db88c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/220e548a4d0f47f7aea2cede713db88c 2024-12-08T03:59:48,409 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/220e548a4d0f47f7aea2cede713db88c, entries=150, sequenceid=533, filesize=12.0 K 2024-12-08T03:59:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a15f06783ece426faf3e381f237d859e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a15f06783ece426faf3e381f237d859e 2024-12-08T03:59:48,417 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a15f06783ece426faf3e381f237d859e, entries=150, sequenceid=533, filesize=12.0 K 2024-12-08T03:59:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/2d851ccf2b7148c898d356512ef66628 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2d851ccf2b7148c898d356512ef66628 2024-12-08T03:59:48,428 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2d851ccf2b7148c898d356512ef66628, entries=150, sequenceid=533, filesize=12.0 K 2024-12-08T03:59:48,428 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 154ms, sequenceid=533, compaction requested=false 2024-12-08T03:59:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-08T03:59:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-08T03:59:48,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-08T03:59:48,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 770 msec 2024-12-08T03:59:48,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 774 msec 2024-12-08T03:59:48,706 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/5cbe6ed1777a4edda9b01d874d568046 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cbe6ed1777a4edda9b01d874d568046 2024-12-08T03:59:48,712 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 5cbe6ed1777a4edda9b01d874d568046(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:48,712 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:48,712 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630388263; duration=0sec 2024-12-08T03:59:48,712 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:48,712 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T03:59:48,798 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:48,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-08T03:59:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T03:59:48,801 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:48,801 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:48,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T03:59:48,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T03:59:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-08T03:59:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-08T03:59:48,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-08T03:59:48,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-12-08T03:59:48,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 164 msec 2024-12-08T03:59:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T03:59:49,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:49,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-08T03:59:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:59:49,120 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:49,121 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:49,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:49,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:49,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:49,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/a229652e652f403b911cf2d3722d67c8 is 50, key is test_row_0/A:col10/1733630389142/Put/seqid=0 2024-12-08T03:59:49,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742114_1290 (size=14741) 2024-12-08T03:59:49,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/a229652e652f403b911cf2d3722d67c8 2024-12-08T03:59:49,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6fd8a6ee831d422095fa15c23134906e is 50, key is test_row_0/B:col10/1733630389142/Put/seqid=0 2024-12-08T03:59:49,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742115_1291 (size=12301) 2024-12-08T03:59:49,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6fd8a6ee831d422095fa15c23134906e 2024-12-08T03:59:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630389185, max timestamp=1733630389217Num uniques 859; 2024-12-08T03:59:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630389185, max timestamp=1733630389217Num uniques 859; 2024-12-08T03:59:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630389185, max timestamp=1733630389217Num uniques 859; 2024-12-08T03:59:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:49,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T03:59:49,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/5ca89c09f6c94ec88126e4a5d6252d7b is 50, key is test_row_0/C:col10/1733630389142/Put/seqid=0 2024-12-08T03:59:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:59:49,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742116_1292 (size=12301) 2024-12-08T03:59:49,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/5ca89c09f6c94ec88126e4a5d6252d7b 2024-12-08T03:59:49,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/a229652e652f403b911cf2d3722d67c8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a229652e652f403b911cf2d3722d67c8 2024-12-08T03:59:49,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a229652e652f403b911cf2d3722d67c8, entries=200, sequenceid=547, filesize=14.4 K 2024-12-08T03:59:49,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6fd8a6ee831d422095fa15c23134906e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6fd8a6ee831d422095fa15c23134906e 2024-12-08T03:59:49,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6fd8a6ee831d422095fa15c23134906e, entries=150, sequenceid=547, filesize=12.0 K 2024-12-08T03:59:49,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/5ca89c09f6c94ec88126e4a5d6252d7b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5ca89c09f6c94ec88126e4a5d6252d7b 2024-12-08T03:59:49,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5ca89c09f6c94ec88126e4a5d6252d7b, entries=150, sequenceid=547, filesize=12.0 K 2024-12-08T03:59:49,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=80.51 KB/82440 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 127ms, sequenceid=547, compaction requested=true 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:49,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:49,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:49,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=87.22 KB heapSize=245.53 KB 2024-12-08T03:59:49,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:49,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:49,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:49,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:49,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:49,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T03:59:49,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:49,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:49,274 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,274 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fb51907d5df74cb7989faaa585ab8c36, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a15f06783ece426faf3e381f237d859e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6fd8a6ee831d422095fa15c23134906e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.7 K 2024-12-08T03:59:49,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:49,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:49,274 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,274 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cbe6ed1777a4edda9b01d874d568046, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/220e548a4d0f47f7aea2cede713db88c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a229652e652f403b911cf2d3722d67c8] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=39.1 K 2024-12-08T03:59:49,274 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting fb51907d5df74cb7989faaa585ab8c36, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733630385469 2024-12-08T03:59:49,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbe6ed1777a4edda9b01d874d568046, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733630385469 2024-12-08T03:59:49,275 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a15f06783ece426faf3e381f237d859e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733630386955 2024-12-08T03:59:49,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:49,276 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd8a6ee831d422095fa15c23134906e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733630389139 2024-12-08T03:59:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 220e548a4d0f47f7aea2cede713db88c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733630386955 2024-12-08T03:59:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting a229652e652f403b911cf2d3722d67c8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733630389128 2024-12-08T03:59:49,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8ba8d40cbe104f8f835cd5a339c2bc4a is 50, key is test_row_0/A:col10/1733630389271/Put/seqid=0 2024-12-08T03:59:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,298 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:49,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/7215db647bd848bb9d132315fe92d8ec is 50, key is test_row_0/B:col10/1733630389142/Put/seqid=0 2024-12-08T03:59:49,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742117_1293 (size=12301) 2024-12-08T03:59:49,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8ba8d40cbe104f8f835cd5a339c2bc4a 2024-12-08T03:59:49,304 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:49,305 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7b1404e3ba4943c285ff9ebe8692eded is 50, key is test_row_0/A:col10/1733630389142/Put/seqid=0 2024-12-08T03:59:49,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742118_1294 (size=13085) 2024-12-08T03:59:49,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6227c2ca5b524867939ac5e0a55f7c36 is 50, key is test_row_0/B:col10/1733630389271/Put/seqid=0 2024-12-08T03:59:49,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742119_1295 (size=13085) 2024-12-08T03:59:49,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742120_1296 (size=12301) 2024-12-08T03:59:49,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6227c2ca5b524867939ac5e0a55f7c36 2024-12-08T03:59:49,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/37262e3c1e9541f1bfcfd0b22dbf968d is 50, key is test_row_0/C:col10/1733630389271/Put/seqid=0 2024-12-08T03:59:49,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742121_1297 (size=12301) 2024-12-08T03:59:49,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/37262e3c1e9541f1bfcfd0b22dbf968d 2024-12-08T03:59:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/8ba8d40cbe104f8f835cd5a339c2bc4a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8ba8d40cbe104f8f835cd5a339c2bc4a 2024-12-08T03:59:49,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8ba8d40cbe104f8f835cd5a339c2bc4a, entries=150, sequenceid=577, filesize=12.0 K 2024-12-08T03:59:49,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6227c2ca5b524867939ac5e0a55f7c36 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6227c2ca5b524867939ac5e0a55f7c36 2024-12-08T03:59:49,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6227c2ca5b524867939ac5e0a55f7c36, entries=150, sequenceid=577, filesize=12.0 K 2024-12-08T03:59:49,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/37262e3c1e9541f1bfcfd0b22dbf968d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/37262e3c1e9541f1bfcfd0b22dbf968d 2024-12-08T03:59:49,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/37262e3c1e9541f1bfcfd0b22dbf968d, entries=150, sequenceid=577, filesize=12.0 K 2024-12-08T03:59:49,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~245.48 KB/251376, currentSize=87.22 KB/89310 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 84ms, sequenceid=577, compaction requested=true 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:49,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-08T03:59:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:49,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T03:59:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:49,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/d4894c7913a048b5802b822025618a3d is 50, key is test_row_0/A:col10/1733630389350/Put/seqid=0 2024-12-08T03:59:49,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742122_1298 (size=12301) 2024-12-08T03:59:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:59:49,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:49,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:49,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630449439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630449547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49448 deadline: 1733630449647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,650 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,711 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/7215db647bd848bb9d132315fe92d8ec as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7215db647bd848bb9d132315fe92d8ec 2024-12-08T03:59:49,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49482 deadline: 1733630449707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,712 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49464 deadline: 1733630449717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,718 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,722 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into 7215db647bd848bb9d132315fe92d8ec(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:49,722 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:49,722 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630389272; duration=0sec 2024-12-08T03:59:49,722 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-08T03:59:49,722 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:49,722 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-08T03:59:49,723 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7b1404e3ba4943c285ff9ebe8692eded as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7b1404e3ba4943c285ff9ebe8692eded 2024-12-08T03:59:49,723 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T03:59:49,723 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T03:59:49,724 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. because compaction request was cancelled 2024-12-08T03:59:49,724 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:49,724 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:49,724 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:49,725 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:49,725 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,725 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/a4b8715d4ec34198bb656f5a171e4632, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2d851ccf2b7148c898d356512ef66628, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5ca89c09f6c94ec88126e4a5d6252d7b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/37262e3c1e9541f1bfcfd0b22dbf968d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=48.7 K 2024-12-08T03:59:49,725 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b8715d4ec34198bb656f5a171e4632, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733630385469 2024-12-08T03:59:49,725 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d851ccf2b7148c898d356512ef66628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733630386955 2024-12-08T03:59:49,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca89c09f6c94ec88126e4a5d6252d7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733630389139 2024-12-08T03:59:49,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 37262e3c1e9541f1bfcfd0b22dbf968d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733630389245 2024-12-08T03:59:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49440 deadline: 1733630449727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,728 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 7b1404e3ba4943c285ff9ebe8692eded(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:49,732 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:49,732 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630389272; duration=0sec 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. because compaction request was cancelled 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:49,733 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in eaa9250e0e196d2f3604a9c3f7ea0f7e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T03:59:49,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T03:59:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,735 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T03:59:49,735 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T03:59:49,735 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. because compaction request was cancelled 2024-12-08T03:59:49,735 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:49,741 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:49,742 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c203524766c34856b3b78f35cc55f258 is 50, key is test_row_0/C:col10/1733630389271/Put/seqid=0 2024-12-08T03:59:49,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:49,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:49,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742123_1299 (size=13119) 2024-12-08T03:59:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:59:49,754 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/c203524766c34856b3b78f35cc55f258 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c203524766c34856b3b78f35cc55f258 2024-12-08T03:59:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630449757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:49,761 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into c203524766c34856b3b78f35cc55f258(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:49,761 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:49,761 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=12, startTime=1733630389358; duration=0sec 2024-12-08T03:59:49,761 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:49,761 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:49,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/d4894c7913a048b5802b822025618a3d 2024-12-08T03:59:49,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b6acd61aa1644338b4f931c6569f2520 is 50, key is test_row_0/B:col10/1733630389350/Put/seqid=0 2024-12-08T03:59:49,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742124_1300 (size=12301) 2024-12-08T03:59:49,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b6acd61aa1644338b4f931c6569f2520 2024-12-08T03:59:49,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/4cbba6a0117847c3bf55f04be4501453 is 50, key is test_row_0/C:col10/1733630389350/Put/seqid=0 2024-12-08T03:59:49,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742125_1301 (size=12301) 2024-12-08T03:59:49,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:49,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:50,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630450067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:50,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/4cbba6a0117847c3bf55f04be4501453 2024-12-08T03:59:50,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/d4894c7913a048b5802b822025618a3d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d4894c7913a048b5802b822025618a3d 2024-12-08T03:59:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:50,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d4894c7913a048b5802b822025618a3d, entries=150, sequenceid=594, filesize=12.0 K 2024-12-08T03:59:50,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/b6acd61aa1644338b4f931c6569f2520 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b6acd61aa1644338b4f931c6569f2520 2024-12-08T03:59:50,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b6acd61aa1644338b4f931c6569f2520, entries=150, sequenceid=594, filesize=12.0 K 2024-12-08T03:59:50,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/4cbba6a0117847c3bf55f04be4501453 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/4cbba6a0117847c3bf55f04be4501453 2024-12-08T03:59:50,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/4cbba6a0117847c3bf55f04be4501453, entries=150, sequenceid=594, filesize=12.0 K 2024-12-08T03:59:50,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 864ms, sequenceid=594, compaction requested=true 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:50,224 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:50,224 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:50,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:50,225 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:50,225 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:50,225 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:50,225 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:50,225 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,225 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,225 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7b1404e3ba4943c285ff9ebe8692eded, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8ba8d40cbe104f8f835cd5a339c2bc4a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d4894c7913a048b5802b822025618a3d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.8 K 2024-12-08T03:59:50,225 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7215db647bd848bb9d132315fe92d8ec, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6227c2ca5b524867939ac5e0a55f7c36, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b6acd61aa1644338b4f931c6569f2520] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.8 K 2024-12-08T03:59:50,226 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b1404e3ba4943c285ff9ebe8692eded, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733630389139 2024-12-08T03:59:50,226 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 7215db647bd848bb9d132315fe92d8ec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733630389139 2024-12-08T03:59:50,226 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ba8d40cbe104f8f835cd5a339c2bc4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733630389245 2024-12-08T03:59:50,226 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6227c2ca5b524867939ac5e0a55f7c36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733630389245 2024-12-08T03:59:50,226 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4894c7913a048b5802b822025618a3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1733630389350 2024-12-08T03:59:50,226 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b6acd61aa1644338b4f931c6569f2520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1733630389350 2024-12-08T03:59:50,234 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:50,235 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/5e8b95f1bdad4c61a6df0fbe26429314 is 50, key is test_row_0/B:col10/1733630389350/Put/seqid=0 2024-12-08T03:59:50,237 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:50,237 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/6f8c1b5e016345a196801842c94d4382 is 50, key is test_row_0/A:col10/1733630389350/Put/seqid=0 2024-12-08T03:59:50,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742126_1302 (size=13187) 2024-12-08T03:59:50,252 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/5e8b95f1bdad4c61a6df0fbe26429314 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/5e8b95f1bdad4c61a6df0fbe26429314 2024-12-08T03:59:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:59:50,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742127_1303 (size=13187) 2024-12-08T03:59:50,267 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into 5e8b95f1bdad4c61a6df0fbe26429314(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:50,267 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:50,267 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/6f8c1b5e016345a196801842c94d4382 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6f8c1b5e016345a196801842c94d4382 2024-12-08T03:59:50,267 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630390224; duration=0sec 2024-12-08T03:59:50,267 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:50,267 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:50,267 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T03:59:50,268 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T03:59:50,268 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T03:59:50,268 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. because compaction request was cancelled 2024-12-08T03:59:50,268 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:50,272 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 6f8c1b5e016345a196801842c94d4382(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:50,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:50,272 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630390224; duration=0sec 2024-12-08T03:59:50,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:50,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:50,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T03:59:50,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:50,363 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T03:59:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:50,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/33233de834584a2bb32c40dbc0229c1b is 50, key is test_row_0/A:col10/1733630389432/Put/seqid=0 2024-12-08T03:59:50,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742128_1304 (size=12301) 2024-12-08T03:59:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:50,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:50,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630450640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:50,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630450747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:50,789 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/33233de834584a2bb32c40dbc0229c1b 2024-12-08T03:59:50,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6c0f9619dc2445b9a1e941b2bd7e36c9 is 50, key is test_row_0/B:col10/1733630389432/Put/seqid=0 2024-12-08T03:59:50,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742129_1305 (size=12301) 2024-12-08T03:59:50,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630450957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:51,216 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6c0f9619dc2445b9a1e941b2bd7e36c9 2024-12-08T03:59:51,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/35928cf049ee4e6ebc7a5f07ce783c4e is 50, key is test_row_0/C:col10/1733630389432/Put/seqid=0 2024-12-08T03:59:51,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742130_1306 (size=12301) 2024-12-08T03:59:51,227 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/35928cf049ee4e6ebc7a5f07ce783c4e 2024-12-08T03:59:51,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/33233de834584a2bb32c40dbc0229c1b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/33233de834584a2bb32c40dbc0229c1b 2024-12-08T03:59:51,244 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/33233de834584a2bb32c40dbc0229c1b, entries=150, sequenceid=614, filesize=12.0 K 2024-12-08T03:59:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/6c0f9619dc2445b9a1e941b2bd7e36c9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6c0f9619dc2445b9a1e941b2bd7e36c9 2024-12-08T03:59:51,251 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6c0f9619dc2445b9a1e941b2bd7e36c9, entries=150, sequenceid=614, filesize=12.0 K 2024-12-08T03:59:51,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/35928cf049ee4e6ebc7a5f07ce783c4e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/35928cf049ee4e6ebc7a5f07ce783c4e 2024-12-08T03:59:51,257 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/35928cf049ee4e6ebc7a5f07ce783c4e, entries=150, sequenceid=614, filesize=12.0 K 2024-12-08T03:59:51,258 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 895ms, sequenceid=614, compaction requested=true 2024-12-08T03:59:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-08T03:59:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-08T03:59:51,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-08T03:59:51,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-12-08T03:59:51,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.1440 sec 2024-12-08T03:59:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T03:59:51,268 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:51,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-08T03:59:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:51,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T03:59:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:51,271 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:51,273 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:51,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:59:51,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/17467d059ca14ad08251ae9ea0e49db1 is 50, key is test_row_0/A:col10/1733630391269/Put/seqid=0 2024-12-08T03:59:51,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742131_1307 (size=14741) 2024-12-08T03:59:51,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/17467d059ca14ad08251ae9ea0e49db1 2024-12-08T03:59:51,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a90be45455444dbda61058cab9deca80 is 50, key is test_row_0/B:col10/1733630391269/Put/seqid=0 2024-12-08T03:59:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 350 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630451324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:51,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742132_1308 (size=12301) 2024-12-08T03:59:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:59:51,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T03:59:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630451437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:51,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T03:59:51,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:59:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630451647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:51,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T03:59:51,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:51,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:51,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a90be45455444dbda61058cab9deca80 2024-12-08T03:59:51,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/5cd4acff677a463e8f23cc7849dc0c37 is 50, key is test_row_0/C:col10/1733630391269/Put/seqid=0 2024-12-08T03:59:51,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742133_1309 (size=12301) 2024-12-08T03:59:51,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/5cd4acff677a463e8f23cc7849dc0c37 2024-12-08T03:59:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/17467d059ca14ad08251ae9ea0e49db1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/17467d059ca14ad08251ae9ea0e49db1 2024-12-08T03:59:51,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/17467d059ca14ad08251ae9ea0e49db1, entries=200, sequenceid=632, filesize=14.4 K 2024-12-08T03:59:51,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/a90be45455444dbda61058cab9deca80 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a90be45455444dbda61058cab9deca80 2024-12-08T03:59:51,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a90be45455444dbda61058cab9deca80, entries=150, sequenceid=632, filesize=12.0 K 2024-12-08T03:59:51,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/5cd4acff677a463e8f23cc7849dc0c37 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5cd4acff677a463e8f23cc7849dc0c37 2024-12-08T03:59:51,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5cd4acff677a463e8f23cc7849dc0c37, entries=150, sequenceid=632, filesize=12.0 K 2024-12-08T03:59:51,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 555ms, sequenceid=632, compaction requested=true 2024-12-08T03:59:51,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:51,826 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:51,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:51,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:51,826 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:51,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:51,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:51,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:51,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:51,830 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:51,830 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:51,830 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:51,830 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:51,830 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,830 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,830 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/5e8b95f1bdad4c61a6df0fbe26429314, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6c0f9619dc2445b9a1e941b2bd7e36c9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a90be45455444dbda61058cab9deca80] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=36.9 K 2024-12-08T03:59:51,830 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6f8c1b5e016345a196801842c94d4382, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/33233de834584a2bb32c40dbc0229c1b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/17467d059ca14ad08251ae9ea0e49db1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=39.3 K 2024-12-08T03:59:51,831 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8b95f1bdad4c61a6df0fbe26429314, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1733630389350 2024-12-08T03:59:51,831 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f8c1b5e016345a196801842c94d4382, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1733630389350 2024-12-08T03:59:51,833 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0f9619dc2445b9a1e941b2bd7e36c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1733630389395 2024-12-08T03:59:51,833 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33233de834584a2bb32c40dbc0229c1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1733630389395 2024-12-08T03:59:51,833 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a90be45455444dbda61058cab9deca80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1733630390630 2024-12-08T03:59:51,833 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17467d059ca14ad08251ae9ea0e49db1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1733630390630 2024-12-08T03:59:51,849 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:51,849 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/7a0a678601a34b88ab84cccca37a279d is 50, key is test_row_0/B:col10/1733630391269/Put/seqid=0 2024-12-08T03:59:51,857 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:51,857 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/52bcf075033341c3983db1a171de45b6 is 50, key is test_row_0/A:col10/1733630391269/Put/seqid=0 2024-12-08T03:59:51,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742135_1311 (size=13289) 2024-12-08T03:59:51,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742134_1310 (size=13289) 2024-12-08T03:59:51,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-08T03:59:51,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,890 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T03:59:51,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:51,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:51,894 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/7a0a678601a34b88ab84cccca37a279d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7a0a678601a34b88ab84cccca37a279d 2024-12-08T03:59:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:59:51,903 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into 7a0a678601a34b88ab84cccca37a279d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:51,903 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:51,903 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630391826; duration=0sec 2024-12-08T03:59:51,903 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:51,903 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:51,903 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T03:59:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/a401aa6f8aa146808c247222d41bc515 is 50, key is test_row_0/A:col10/1733630391300/Put/seqid=0 2024-12-08T03:59:51,911 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T03:59:51,911 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:51,911 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:51,911 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c203524766c34856b3b78f35cc55f258, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/4cbba6a0117847c3bf55f04be4501453, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/35928cf049ee4e6ebc7a5f07ce783c4e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5cd4acff677a463e8f23cc7849dc0c37] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=48.8 K 2024-12-08T03:59:51,912 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c203524766c34856b3b78f35cc55f258, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733630389245 2024-12-08T03:59:51,912 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cbba6a0117847c3bf55f04be4501453, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1733630389350 2024-12-08T03:59:51,912 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 35928cf049ee4e6ebc7a5f07ce783c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1733630389395 2024-12-08T03:59:51,912 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd4acff677a463e8f23cc7849dc0c37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1733630390630 2024-12-08T03:59:51,941 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:51,942 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8f4569f954944d5d9a1b3fdf167d8c31 is 50, key is test_row_0/C:col10/1733630391269/Put/seqid=0 2024-12-08T03:59:51,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:51,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742136_1312 (size=12301) 2024-12-08T03:59:51,973 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/a401aa6f8aa146808c247222d41bc515 2024-12-08T03:59:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/24912a8ec69948d0ab3608ad918904ee is 50, key is test_row_0/B:col10/1733630391300/Put/seqid=0 2024-12-08T03:59:51,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742137_1313 (size=13255) 2024-12-08T03:59:52,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742138_1314 (size=12301) 2024-12-08T03:59:52,001 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/24912a8ec69948d0ab3608ad918904ee 2024-12-08T03:59:52,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/88d44da6a6cd4b3e9e5bc1d135b5f602 is 50, key is test_row_0/C:col10/1733630391300/Put/seqid=0 2024-12-08T03:59:52,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742139_1315 (size=12301) 2024-12-08T03:59:52,021 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/88d44da6a6cd4b3e9e5bc1d135b5f602 2024-12-08T03:59:52,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/a401aa6f8aa146808c247222d41bc515 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a401aa6f8aa146808c247222d41bc515 2024-12-08T03:59:52,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:52,031 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a401aa6f8aa146808c247222d41bc515, entries=150, sequenceid=646, filesize=12.0 K 2024-12-08T03:59:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630452027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/24912a8ec69948d0ab3608ad918904ee as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/24912a8ec69948d0ab3608ad918904ee 2024-12-08T03:59:52,041 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/24912a8ec69948d0ab3608ad918904ee, entries=150, sequenceid=646, filesize=12.0 K 2024-12-08T03:59:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/88d44da6a6cd4b3e9e5bc1d135b5f602 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/88d44da6a6cd4b3e9e5bc1d135b5f602 2024-12-08T03:59:52,049 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/88d44da6a6cd4b3e9e5bc1d135b5f602, entries=150, sequenceid=646, filesize=12.0 K 2024-12-08T03:59:52,050 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 160ms, sequenceid=646, compaction requested=false 2024-12-08T03:59:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-08T03:59:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-08T03:59:52,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-08T03:59:52,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 779 msec 2024-12-08T03:59:52,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 787 msec 2024-12-08T03:59:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:52,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T03:59:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:52,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:52,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/b3efcb415e824548a053e4a1493d4051 is 50, key is test_row_0/A:col10/1733630392015/Put/seqid=0 2024-12-08T03:59:52,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742140_1316 (size=14741) 2024-12-08T03:59:52,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/b3efcb415e824548a053e4a1493d4051 2024-12-08T03:59:52,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 380 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630452199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:52,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0b88b470344b4b6a9165b39ffe387bd7 is 50, key is test_row_0/B:col10/1733630392015/Put/seqid=0 2024-12-08T03:59:52,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742141_1317 (size=12301) 2024-12-08T03:59:52,292 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/52bcf075033341c3983db1a171de45b6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/52bcf075033341c3983db1a171de45b6 2024-12-08T03:59:52,298 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into 52bcf075033341c3983db1a171de45b6(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:52,298 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:52,298 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630391826; duration=0sec 2024-12-08T03:59:52,298 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:52,298 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 381 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630452307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:52,399 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8f4569f954944d5d9a1b3fdf167d8c31 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8f4569f954944d5d9a1b3fdf167d8c31 2024-12-08T03:59:52,405 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into 8f4569f954944d5d9a1b3fdf167d8c31(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:52,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:52,405 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=12, startTime=1733630391827; duration=0sec 2024-12-08T03:59:52,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:52,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T03:59:52,407 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:52,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T03:59:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-08T03:59:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:52,410 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T03:59:52,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T03:59:52,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T03:59:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T03:59:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49486 deadline: 1733630452517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T03:59:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T03:59:52,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0b88b470344b4b6a9165b39ffe387bd7 2024-12-08T03:59:52,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/160eb164ae2b4cdd978db2f6471ce18f is 50, key is test_row_0/C:col10/1733630392015/Put/seqid=0 2024-12-08T03:59:52,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742142_1318 (size=12301) 2024-12-08T03:59:52,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/160eb164ae2b4cdd978db2f6471ce18f 2024-12-08T03:59:52,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/b3efcb415e824548a053e4a1493d4051 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/b3efcb415e824548a053e4a1493d4051 2024-12-08T03:59:52,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/b3efcb415e824548a053e4a1493d4051, entries=200, sequenceid=666, filesize=14.4 K 2024-12-08T03:59:52,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/0b88b470344b4b6a9165b39ffe387bd7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0b88b470344b4b6a9165b39ffe387bd7 2024-12-08T03:59:52,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0b88b470344b4b6a9165b39ffe387bd7, entries=150, sequenceid=666, filesize=12.0 K 2024-12-08T03:59:52,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/160eb164ae2b4cdd978db2f6471ce18f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/160eb164ae2b4cdd978db2f6471ce18f 2024-12-08T03:59:52,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:52,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:52,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/160eb164ae2b4cdd978db2f6471ce18f, entries=150, sequenceid=666, filesize=12.0 K 2024-12-08T03:59:52,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 589ms, sequenceid=666, compaction requested=true 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:52,730 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:52,730 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eaa9250e0e196d2f3604a9c3f7ea0f7e:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T03:59:52,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:52,731 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:52,731 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/B is initiating minor compaction (all files) 2024-12-08T03:59:52,731 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/B in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,731 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:52,731 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7a0a678601a34b88ab84cccca37a279d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/24912a8ec69948d0ab3608ad918904ee, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0b88b470344b4b6a9165b39ffe387bd7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=37.0 K 2024-12-08T03:59:52,731 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/A is initiating minor compaction (all files) 2024-12-08T03:59:52,731 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/A in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,731 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/52bcf075033341c3983db1a171de45b6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a401aa6f8aa146808c247222d41bc515, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/b3efcb415e824548a053e4a1493d4051] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=39.4 K 2024-12-08T03:59:52,732 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a0a678601a34b88ab84cccca37a279d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1733630390630 2024-12-08T03:59:52,732 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 24912a8ec69948d0ab3608ad918904ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733630391300 2024-12-08T03:59:52,732 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52bcf075033341c3983db1a171de45b6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1733630390630 2024-12-08T03:59:52,732 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b88b470344b4b6a9165b39ffe387bd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733630392011 2024-12-08T03:59:52,732 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting a401aa6f8aa146808c247222d41bc515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733630391300 2024-12-08T03:59:52,733 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3efcb415e824548a053e4a1493d4051, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733630392011 2024-12-08T03:59:52,755 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#B#compaction#275 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:52,755 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/afed4c1de84945b1982f60eb5264ef34 is 50, key is test_row_0/B:col10/1733630392015/Put/seqid=0 2024-12-08T03:59:52,777 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:52,777 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/d23c0439091342e7a35ce2f71f7b7ef3 is 50, key is test_row_0/A:col10/1733630392015/Put/seqid=0 2024-12-08T03:59:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:52,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T03:59:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742143_1319 (size=13391) 2024-12-08T03:59:52,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742144_1320 (size=13391) 2024-12-08T03:59:52,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/63035356972442608dd47e69bbdb1b36 is 50, key is test_row_0/A:col10/1733630392830/Put/seqid=0 2024-12-08T03:59:52,873 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-12-08T03:59:52,873 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:52,873 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,873 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,873 INFO [Thread-1063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1063. 2024-12-08T03:59:52,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:52,873 DEBUG [Thread-1063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:52,873 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,874 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:52,875 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-12-08T03:59:52,875 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:52,875 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,875 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:52,877 INFO [Thread-1061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1061. 2024-12-08T03:59:52,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:52,877 DEBUG [Thread-1061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:52,877 DEBUG [Thread-1061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,877 DEBUG [Thread-1061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:52,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:52,879 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-12-08T03:59:52,879 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:52,879 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,879 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:52,881 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-12-08T03:59:52,881 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:52,881 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,881 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:52,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:52,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742145_1321 (size=14741) 2024-12-08T03:59:53,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:53,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:53,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:53,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,238 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/afed4c1de84945b1982f60eb5264ef34 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/afed4c1de84945b1982f60eb5264ef34 2024-12-08T03:59:53,244 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/B of eaa9250e0e196d2f3604a9c3f7ea0f7e into afed4c1de84945b1982f60eb5264ef34(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:53,245 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:53,245 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/B, priority=13, startTime=1733630392730; duration=0sec 2024-12-08T03:59:53,245 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T03:59:53,245 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:B 2024-12-08T03:59:53,245 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T03:59:53,246 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T03:59:53,246 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): eaa9250e0e196d2f3604a9c3f7ea0f7e/C is initiating minor compaction (all files) 2024-12-08T03:59:53,246 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eaa9250e0e196d2f3604a9c3f7ea0f7e/C in TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,246 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8f4569f954944d5d9a1b3fdf167d8c31, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/88d44da6a6cd4b3e9e5bc1d135b5f602, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/160eb164ae2b4cdd978db2f6471ce18f] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp, totalSize=37.0 K 2024-12-08T03:59:53,247 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f4569f954944d5d9a1b3fdf167d8c31, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1733630390630 2024-12-08T03:59:53,247 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d44da6a6cd4b3e9e5bc1d135b5f602, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1733630391300 2024-12-08T03:59:53,247 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 160eb164ae2b4cdd978db2f6471ce18f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733630392011 2024-12-08T03:59:53,256 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eaa9250e0e196d2f3604a9c3f7ea0f7e#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T03:59:53,256 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/30b51394e0f74e3a97817a5299b52a32 is 50, key is test_row_0/C:col10/1733630392015/Put/seqid=0 2024-12-08T03:59:53,262 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/d23c0439091342e7a35ce2f71f7b7ef3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d23c0439091342e7a35ce2f71f7b7ef3 2024-12-08T03:59:53,268 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/A of eaa9250e0e196d2f3604a9c3f7ea0f7e into d23c0439091342e7a35ce2f71f7b7ef3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:53,268 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:53,268 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/A, priority=13, startTime=1733630392730; duration=0sec 2024-12-08T03:59:53,269 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:53,269 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:A 2024-12-08T03:59:53,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742146_1322 (size=13357) 2024-12-08T03:59:53,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/63035356972442608dd47e69bbdb1b36 2024-12-08T03:59:53,289 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/30b51394e0f74e3a97817a5299b52a32 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/30b51394e0f74e3a97817a5299b52a32 2024-12-08T03:59:53,299 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eaa9250e0e196d2f3604a9c3f7ea0f7e/C of eaa9250e0e196d2f3604a9c3f7ea0f7e into 30b51394e0f74e3a97817a5299b52a32(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T03:59:53,299 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:53,299 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., storeName=eaa9250e0e196d2f3604a9c3f7ea0f7e/C, priority=13, startTime=1733630392730; duration=0sec 2024-12-08T03:59:53,300 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T03:59:53,300 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eaa9250e0e196d2f3604a9c3f7ea0f7e:C 2024-12-08T03:59:53,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/8dbba20d46694135b79d92c9e2bdf02a is 50, key is test_row_0/B:col10/1733630392830/Put/seqid=0 2024-12-08T03:59:53,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T03:59:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742147_1323 (size=12301) 2024-12-08T03:59:53,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/8dbba20d46694135b79d92c9e2bdf02a 2024-12-08T03:59:53,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/fb129e7b51fb4efda6a10cc42b491ead is 50, key is test_row_0/C:col10/1733630392830/Put/seqid=0 2024-12-08T03:59:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742148_1324 (size=12301) 2024-12-08T03:59:53,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/fb129e7b51fb4efda6a10cc42b491ead 2024-12-08T03:59:53,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/63035356972442608dd47e69bbdb1b36 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/63035356972442608dd47e69bbdb1b36 2024-12-08T03:59:53,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/63035356972442608dd47e69bbdb1b36, entries=200, sequenceid=682, filesize=14.4 K 2024-12-08T03:59:53,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/8dbba20d46694135b79d92c9e2bdf02a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8dbba20d46694135b79d92c9e2bdf02a 2024-12-08T03:59:53,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8dbba20d46694135b79d92c9e2bdf02a, entries=150, sequenceid=682, filesize=12.0 K 2024-12-08T03:59:53,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/fb129e7b51fb4efda6a10cc42b491ead as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb129e7b51fb4efda6a10cc42b491ead 2024-12-08T03:59:53,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb129e7b51fb4efda6a10cc42b491ead, entries=150, sequenceid=682, filesize=12.0 K 2024-12-08T03:59:53,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=46.96 KB/48090 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 588ms, sequenceid=682, compaction requested=false 2024-12-08T03:59:53,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:53,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-08T03:59:53,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:53,502 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T03:59:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:53,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:53,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/df01567b39c04fd78c1b2d21846429a4 is 50, key is test_row_0/A:col10/1733630392854/Put/seqid=0 2024-12-08T03:59:53,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742149_1325 (size=12301) 2024-12-08T03:59:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. as already flushing 2024-12-08T03:59:53,689 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-12-08T03:59:53,689 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:53,689 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:53,689 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:53,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:53,912 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/df01567b39c04fd78c1b2d21846429a4 2024-12-08T03:59:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/37b3c2bb464f48c1a905dacdc5c07531 is 50, key is test_row_0/B:col10/1733630392854/Put/seqid=0 2024-12-08T03:59:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742150_1326 (size=12301) 2024-12-08T03:59:54,324 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/37b3c2bb464f48c1a905dacdc5c07531 2024-12-08T03:59:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8568cea969e74c08be7ba7b038dea9e7 is 50, key is test_row_0/C:col10/1733630392854/Put/seqid=0 2024-12-08T03:59:54,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742151_1327 (size=12301) 2024-12-08T03:59:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:54,742 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8568cea969e74c08be7ba7b038dea9e7 2024-12-08T03:59:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/df01567b39c04fd78c1b2d21846429a4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/df01567b39c04fd78c1b2d21846429a4 2024-12-08T03:59:54,753 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/df01567b39c04fd78c1b2d21846429a4, entries=150, sequenceid=695, filesize=12.0 K 2024-12-08T03:59:54,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/37b3c2bb464f48c1a905dacdc5c07531 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/37b3c2bb464f48c1a905dacdc5c07531 2024-12-08T03:59:54,758 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/37b3c2bb464f48c1a905dacdc5c07531, entries=150, sequenceid=695, filesize=12.0 K 2024-12-08T03:59:54,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/8568cea969e74c08be7ba7b038dea9e7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8568cea969e74c08be7ba7b038dea9e7 2024-12-08T03:59:54,763 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8568cea969e74c08be7ba7b038dea9e7, entries=150, sequenceid=695, filesize=12.0 K 2024-12-08T03:59:54,764 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 1262ms, sequenceid=695, compaction requested=true 2024-12-08T03:59:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: 2024-12-08T03:59:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-08T03:59:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-08T03:59:54,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-08T03:59:54,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-12-08T03:59:54,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.3610 sec 2024-12-08T03:59:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-08T03:59:56,587 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T03:59:56,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T03:59:59,758 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-12-08T03:59:59,758 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:59,758 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,758 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:59,769 INFO [Thread-1050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1050. 2024-12-08T03:59:59,769 DEBUG [Thread-1050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:59,769 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,769 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-12-08T03:59:59,769 DEBUG [Thread-1050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,769 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T03:59:59,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:59,769 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,769 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 340 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3984 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3719 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3775 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3979 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3699 2024-12-08T03:59:59,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T03:59:59,770 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T03:59:59,770 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T03:59:59,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T03:59:59,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T03:59:59,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T03:59:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T03:59:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T03:59:59,774 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630399774"}]},"ts":"1733630399774"} 2024-12-08T03:59:59,775 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T03:59:59,775 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T03:59:59,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T03:59:59,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, UNASSIGN}] 2024-12-08T03:59:59,778 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, UNASSIGN 2024-12-08T03:59:59,778 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=eaa9250e0e196d2f3604a9c3f7ea0f7e, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T03:59:59,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, UNASSIGN because future has completed 2024-12-08T03:59:59,780 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T03:59:59,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T03:59:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T03:59:59,932 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T03:59:59,932 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing eaa9250e0e196d2f3604a9c3f7ea0f7e, disabling compactions & flushes 2024-12-08T03:59:59,933 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. after waiting 0 ms 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T03:59:59,933 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=A 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=B 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eaa9250e0e196d2f3604a9c3f7ea0f7e, store=C 2024-12-08T03:59:59,933 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T03:59:59,937 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7950a84d2ee2481ab2ec1cddc9a67fb3 is 50, key is test_row_0/A:col10/1733630399767/Put/seqid=0 2024-12-08T03:59:59,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742152_1328 (size=12301) 2024-12-08T03:59:59,941 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7950a84d2ee2481ab2ec1cddc9a67fb3 2024-12-08T03:59:59,949 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/d16a9079ec574f2d9cf640cf2c41b2f7 is 50, key is test_row_0/B:col10/1733630399767/Put/seqid=0 2024-12-08T03:59:59,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742153_1329 (size=12301) 2024-12-08T04:00:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T04:00:00,369 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/d16a9079ec574f2d9cf640cf2c41b2f7 2024-12-08T04:00:00,376 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/07a4ca9f787848928b8c9c63410db52f is 50, key is test_row_0/C:col10/1733630399767/Put/seqid=0 2024-12-08T04:00:00,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742154_1330 (size=12301) 2024-12-08T04:00:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T04:00:00,780 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/07a4ca9f787848928b8c9c63410db52f 2024-12-08T04:00:00,784 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/A/7950a84d2ee2481ab2ec1cddc9a67fb3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7950a84d2ee2481ab2ec1cddc9a67fb3 2024-12-08T04:00:00,788 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7950a84d2ee2481ab2ec1cddc9a67fb3, entries=150, sequenceid=702, filesize=12.0 K 2024-12-08T04:00:00,789 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/B/d16a9079ec574f2d9cf640cf2c41b2f7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d16a9079ec574f2d9cf640cf2c41b2f7 2024-12-08T04:00:00,793 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d16a9079ec574f2d9cf640cf2c41b2f7, entries=150, sequenceid=702, filesize=12.0 K 2024-12-08T04:00:00,794 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/.tmp/C/07a4ca9f787848928b8c9c63410db52f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/07a4ca9f787848928b8c9c63410db52f 2024-12-08T04:00:00,798 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/07a4ca9f787848928b8c9c63410db52f, entries=150, sequenceid=702, filesize=12.0 K 2024-12-08T04:00:00,799 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 866ms, sequenceid=702, compaction requested=true 2024-12-08T04:00:00,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cfeb19e210e4fb08eb5333a4b0bc1bd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c7d83fb8085a4541aef6e964d56a6f1c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5452bcd61aa040a495242a6c57fe0778, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8c9dc8675fe444c08dbff070a276d427, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/4da5e321c1d644fe8c9291f1e18c726d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8f00e7181eda4616b6269feb2ec3596f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/259fa28bc9c443299226de12a78b7649, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6ea485dbbbd549f6a94ae1064277d07f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7824a337bbc34b8390224c2f0b523675, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/168c22b0cae94f41add12333717d8ea6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7429ca0d41f74a84b0172fd127c5515c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/3d056b462af048c3880e947229abb3b8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/45c6441bb6624600968be30584153a1f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e3485ea8a9c84b0282bfb1a3a2116aaf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/36e9b570c4414d5b86b9615a8f0b17e6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/2f40718aa0674d3e84f60154d2e9e340, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e91671ded731486e9dac252357a18b79, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/90ff65e4b9e04ba480d97ad680053888, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c193534bfa8f451fb9c7be86d718bbb7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cbe6ed1777a4edda9b01d874d568046, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/fb51373d57944999b0fe733524c1eb3a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/220e548a4d0f47f7aea2cede713db88c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a229652e652f403b911cf2d3722d67c8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7b1404e3ba4943c285ff9ebe8692eded, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8ba8d40cbe104f8f835cd5a339c2bc4a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6f8c1b5e016345a196801842c94d4382, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d4894c7913a048b5802b822025618a3d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/33233de834584a2bb32c40dbc0229c1b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/17467d059ca14ad08251ae9ea0e49db1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/52bcf075033341c3983db1a171de45b6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a401aa6f8aa146808c247222d41bc515, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/b3efcb415e824548a053e4a1493d4051] to archive 2024-12-08T04:00:00,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:00:00,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cfeb19e210e4fb08eb5333a4b0bc1bd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cfeb19e210e4fb08eb5333a4b0bc1bd 2024-12-08T04:00:00,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c7d83fb8085a4541aef6e964d56a6f1c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c7d83fb8085a4541aef6e964d56a6f1c 2024-12-08T04:00:00,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5452bcd61aa040a495242a6c57fe0778 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5452bcd61aa040a495242a6c57fe0778 2024-12-08T04:00:00,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8c9dc8675fe444c08dbff070a276d427 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8c9dc8675fe444c08dbff070a276d427 2024-12-08T04:00:00,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/4da5e321c1d644fe8c9291f1e18c726d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/4da5e321c1d644fe8c9291f1e18c726d 2024-12-08T04:00:00,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8f00e7181eda4616b6269feb2ec3596f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8f00e7181eda4616b6269feb2ec3596f 2024-12-08T04:00:00,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/259fa28bc9c443299226de12a78b7649 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/259fa28bc9c443299226de12a78b7649 2024-12-08T04:00:00,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6ea485dbbbd549f6a94ae1064277d07f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6ea485dbbbd549f6a94ae1064277d07f 2024-12-08T04:00:00,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7824a337bbc34b8390224c2f0b523675 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7824a337bbc34b8390224c2f0b523675 2024-12-08T04:00:00,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/168c22b0cae94f41add12333717d8ea6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/168c22b0cae94f41add12333717d8ea6 2024-12-08T04:00:00,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7429ca0d41f74a84b0172fd127c5515c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7429ca0d41f74a84b0172fd127c5515c 2024-12-08T04:00:00,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/3d056b462af048c3880e947229abb3b8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/3d056b462af048c3880e947229abb3b8 2024-12-08T04:00:00,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/45c6441bb6624600968be30584153a1f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/45c6441bb6624600968be30584153a1f 2024-12-08T04:00:00,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e3485ea8a9c84b0282bfb1a3a2116aaf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e3485ea8a9c84b0282bfb1a3a2116aaf 2024-12-08T04:00:00,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/36e9b570c4414d5b86b9615a8f0b17e6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/36e9b570c4414d5b86b9615a8f0b17e6 2024-12-08T04:00:00,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/2f40718aa0674d3e84f60154d2e9e340 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/2f40718aa0674d3e84f60154d2e9e340 2024-12-08T04:00:00,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e91671ded731486e9dac252357a18b79 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/e91671ded731486e9dac252357a18b79 2024-12-08T04:00:00,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/90ff65e4b9e04ba480d97ad680053888 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/90ff65e4b9e04ba480d97ad680053888 2024-12-08T04:00:00,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c193534bfa8f451fb9c7be86d718bbb7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/c193534bfa8f451fb9c7be86d718bbb7 2024-12-08T04:00:00,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cbe6ed1777a4edda9b01d874d568046 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/5cbe6ed1777a4edda9b01d874d568046 2024-12-08T04:00:00,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/fb51373d57944999b0fe733524c1eb3a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/fb51373d57944999b0fe733524c1eb3a 2024-12-08T04:00:00,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/220e548a4d0f47f7aea2cede713db88c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/220e548a4d0f47f7aea2cede713db88c 2024-12-08T04:00:00,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a229652e652f403b911cf2d3722d67c8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a229652e652f403b911cf2d3722d67c8 2024-12-08T04:00:00,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7b1404e3ba4943c285ff9ebe8692eded to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7b1404e3ba4943c285ff9ebe8692eded 2024-12-08T04:00:00,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8ba8d40cbe104f8f835cd5a339c2bc4a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/8ba8d40cbe104f8f835cd5a339c2bc4a 2024-12-08T04:00:00,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6f8c1b5e016345a196801842c94d4382 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/6f8c1b5e016345a196801842c94d4382 2024-12-08T04:00:00,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d4894c7913a048b5802b822025618a3d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d4894c7913a048b5802b822025618a3d 2024-12-08T04:00:00,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/33233de834584a2bb32c40dbc0229c1b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/33233de834584a2bb32c40dbc0229c1b 2024-12-08T04:00:00,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/17467d059ca14ad08251ae9ea0e49db1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/17467d059ca14ad08251ae9ea0e49db1 2024-12-08T04:00:00,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/52bcf075033341c3983db1a171de45b6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/52bcf075033341c3983db1a171de45b6 2024-12-08T04:00:00,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a401aa6f8aa146808c247222d41bc515 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/a401aa6f8aa146808c247222d41bc515 2024-12-08T04:00:00,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/b3efcb415e824548a053e4a1493d4051 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/b3efcb415e824548a053e4a1493d4051 2024-12-08T04:00:00,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d0cdcffd761f4b34b933666b84be938f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a31b202f7561420798efc4757a098d66, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/62b1458f8377411ebd7e3fcdead98e7e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fc45df01e9234c8bb2c2e52bb187fce3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0ab56cff08e84fa9963c383203af0850, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c3a47deb1d0d42f4ac733c150a0ee688, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c52b3b4e8d874bd08ab8bf717f832abd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3e08b1f825b241b4bfb330a86ee34ab5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0706ca4035fa462490c855a8f5f6418d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b0890c3cb2954acf870b6d1c0b529d9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/331afa8390624eaeae1ea851004f125c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/906072e2f0e14c8a816e3ed09f68eef2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3425a22c726c46be8418bb2c294a59d2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/4b16f86142b84d9da17dcba4fccf4105, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/f0fc07940c7e48d29774c2e33e2ca97e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/efa0d3f784324790b48afef626a92b2e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6f1b939d9e2341dbba654b35e38c25f2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/453ccb9f957b4542afcd13c5013c9a21, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fb51907d5df74cb7989faaa585ab8c36, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8d70d3aa62114557b67ee7e4aeee4235, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a15f06783ece426faf3e381f237d859e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7215db647bd848bb9d132315fe92d8ec, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6fd8a6ee831d422095fa15c23134906e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6227c2ca5b524867939ac5e0a55f7c36, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/5e8b95f1bdad4c61a6df0fbe26429314, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b6acd61aa1644338b4f931c6569f2520, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6c0f9619dc2445b9a1e941b2bd7e36c9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7a0a678601a34b88ab84cccca37a279d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a90be45455444dbda61058cab9deca80, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/24912a8ec69948d0ab3608ad918904ee, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0b88b470344b4b6a9165b39ffe387bd7] to archive 2024-12-08T04:00:00,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:00:00,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d0cdcffd761f4b34b933666b84be938f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d0cdcffd761f4b34b933666b84be938f 2024-12-08T04:00:00,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a31b202f7561420798efc4757a098d66 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a31b202f7561420798efc4757a098d66 2024-12-08T04:00:00,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/62b1458f8377411ebd7e3fcdead98e7e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/62b1458f8377411ebd7e3fcdead98e7e 2024-12-08T04:00:00,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fc45df01e9234c8bb2c2e52bb187fce3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fc45df01e9234c8bb2c2e52bb187fce3 2024-12-08T04:00:00,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0ab56cff08e84fa9963c383203af0850 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0ab56cff08e84fa9963c383203af0850 2024-12-08T04:00:00,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c3a47deb1d0d42f4ac733c150a0ee688 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c3a47deb1d0d42f4ac733c150a0ee688 2024-12-08T04:00:00,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b9bc7fa6c4df478b9cb73ecaa3fb45b8 2024-12-08T04:00:00,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c52b3b4e8d874bd08ab8bf717f832abd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/c52b3b4e8d874bd08ab8bf717f832abd 2024-12-08T04:00:00,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3e08b1f825b241b4bfb330a86ee34ab5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3e08b1f825b241b4bfb330a86ee34ab5 2024-12-08T04:00:00,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0706ca4035fa462490c855a8f5f6418d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0706ca4035fa462490c855a8f5f6418d 2024-12-08T04:00:00,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b0890c3cb2954acf870b6d1c0b529d9c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b0890c3cb2954acf870b6d1c0b529d9c 2024-12-08T04:00:00,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/331afa8390624eaeae1ea851004f125c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/331afa8390624eaeae1ea851004f125c 2024-12-08T04:00:00,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/906072e2f0e14c8a816e3ed09f68eef2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/906072e2f0e14c8a816e3ed09f68eef2 2024-12-08T04:00:00,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3425a22c726c46be8418bb2c294a59d2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/3425a22c726c46be8418bb2c294a59d2 2024-12-08T04:00:00,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/4b16f86142b84d9da17dcba4fccf4105 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/4b16f86142b84d9da17dcba4fccf4105 2024-12-08T04:00:00,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/f0fc07940c7e48d29774c2e33e2ca97e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/f0fc07940c7e48d29774c2e33e2ca97e 2024-12-08T04:00:00,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/efa0d3f784324790b48afef626a92b2e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/efa0d3f784324790b48afef626a92b2e 2024-12-08T04:00:00,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6f1b939d9e2341dbba654b35e38c25f2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6f1b939d9e2341dbba654b35e38c25f2 2024-12-08T04:00:00,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/453ccb9f957b4542afcd13c5013c9a21 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/453ccb9f957b4542afcd13c5013c9a21 2024-12-08T04:00:00,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fb51907d5df74cb7989faaa585ab8c36 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/fb51907d5df74cb7989faaa585ab8c36 2024-12-08T04:00:00,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8d70d3aa62114557b67ee7e4aeee4235 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8d70d3aa62114557b67ee7e4aeee4235 2024-12-08T04:00:00,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a15f06783ece426faf3e381f237d859e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a15f06783ece426faf3e381f237d859e 2024-12-08T04:00:00,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7215db647bd848bb9d132315fe92d8ec to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7215db647bd848bb9d132315fe92d8ec 2024-12-08T04:00:00,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6fd8a6ee831d422095fa15c23134906e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6fd8a6ee831d422095fa15c23134906e 2024-12-08T04:00:00,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6227c2ca5b524867939ac5e0a55f7c36 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6227c2ca5b524867939ac5e0a55f7c36 2024-12-08T04:00:00,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/5e8b95f1bdad4c61a6df0fbe26429314 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/5e8b95f1bdad4c61a6df0fbe26429314 2024-12-08T04:00:00,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b6acd61aa1644338b4f931c6569f2520 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/b6acd61aa1644338b4f931c6569f2520 2024-12-08T04:00:00,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6c0f9619dc2445b9a1e941b2bd7e36c9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/6c0f9619dc2445b9a1e941b2bd7e36c9 2024-12-08T04:00:00,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7a0a678601a34b88ab84cccca37a279d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/7a0a678601a34b88ab84cccca37a279d 2024-12-08T04:00:00,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a90be45455444dbda61058cab9deca80 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/a90be45455444dbda61058cab9deca80 2024-12-08T04:00:00,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/24912a8ec69948d0ab3608ad918904ee to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/24912a8ec69948d0ab3608ad918904ee 2024-12-08T04:00:00,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0b88b470344b4b6a9165b39ffe387bd7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/0b88b470344b4b6a9165b39ffe387bd7 2024-12-08T04:00:00,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d0cef19501ba4663b26f0abea2044df3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c8fdbeaac3264828af763e1d44791f06, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f13d93b114bd4856876d563d5cb9fbe5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/6a5862c5965d49f9a2a07281aa83b9b4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c80c1ce53ede447eb55358ac310d9164, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8358f061641947d9bb70e6cc848bd117, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f343af42048d4fbd951a0dea4af44350, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f0297f64ebc549509d370f8ca874febe, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2edef5ef05fa44e7800b31152ac762a1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/db020012c46c4bff90a3e5db70c2d643, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/7c10692647e2406cac7fbe4dfa6ce87b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/ed8b4dfa048246358ade725a483ece21, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb128d9067344d278f65ee7a544fcab6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/3b63367dec8d4f1994c8437109b7454f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/17254057569b442d86108592d2a3aadf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d8e3436c9a6e46d0a5f0117111362c56, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/e98a915cb84b4a7bb3432fe8fad35f1d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2cc73d0cf53545ac94de0c55dacdb4b3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/00af385ff45c4586bdfb2907f72b6d61, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/a4b8715d4ec34198bb656f5a171e4632, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/02c2559cd0024ec3849a79424a638e13, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2d851ccf2b7148c898d356512ef66628, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5ca89c09f6c94ec88126e4a5d6252d7b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c203524766c34856b3b78f35cc55f258, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/37262e3c1e9541f1bfcfd0b22dbf968d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/4cbba6a0117847c3bf55f04be4501453, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/35928cf049ee4e6ebc7a5f07ce783c4e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8f4569f954944d5d9a1b3fdf167d8c31, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5cd4acff677a463e8f23cc7849dc0c37, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/88d44da6a6cd4b3e9e5bc1d135b5f602, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/160eb164ae2b4cdd978db2f6471ce18f] to archive 2024-12-08T04:00:00,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:00:00,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d0cef19501ba4663b26f0abea2044df3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d0cef19501ba4663b26f0abea2044df3 2024-12-08T04:00:00,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c8fdbeaac3264828af763e1d44791f06 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c8fdbeaac3264828af763e1d44791f06 2024-12-08T04:00:00,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f13d93b114bd4856876d563d5cb9fbe5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f13d93b114bd4856876d563d5cb9fbe5 2024-12-08T04:00:00,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/6a5862c5965d49f9a2a07281aa83b9b4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/6a5862c5965d49f9a2a07281aa83b9b4 2024-12-08T04:00:00,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c80c1ce53ede447eb55358ac310d9164 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c80c1ce53ede447eb55358ac310d9164 2024-12-08T04:00:00,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8358f061641947d9bb70e6cc848bd117 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8358f061641947d9bb70e6cc848bd117 2024-12-08T04:00:00,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f343af42048d4fbd951a0dea4af44350 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f343af42048d4fbd951a0dea4af44350 2024-12-08T04:00:00,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f0297f64ebc549509d370f8ca874febe to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/f0297f64ebc549509d370f8ca874febe 2024-12-08T04:00:00,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2edef5ef05fa44e7800b31152ac762a1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2edef5ef05fa44e7800b31152ac762a1 2024-12-08T04:00:00,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/db020012c46c4bff90a3e5db70c2d643 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/db020012c46c4bff90a3e5db70c2d643 2024-12-08T04:00:00,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/7c10692647e2406cac7fbe4dfa6ce87b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/7c10692647e2406cac7fbe4dfa6ce87b 2024-12-08T04:00:00,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/ed8b4dfa048246358ade725a483ece21 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/ed8b4dfa048246358ade725a483ece21 2024-12-08T04:00:00,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb128d9067344d278f65ee7a544fcab6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb128d9067344d278f65ee7a544fcab6 2024-12-08T04:00:00,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/3b63367dec8d4f1994c8437109b7454f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/3b63367dec8d4f1994c8437109b7454f 2024-12-08T04:00:00,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/17254057569b442d86108592d2a3aadf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/17254057569b442d86108592d2a3aadf 2024-12-08T04:00:00,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d8e3436c9a6e46d0a5f0117111362c56 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/d8e3436c9a6e46d0a5f0117111362c56 2024-12-08T04:00:00,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/e98a915cb84b4a7bb3432fe8fad35f1d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/e98a915cb84b4a7bb3432fe8fad35f1d 2024-12-08T04:00:00,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2cc73d0cf53545ac94de0c55dacdb4b3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2cc73d0cf53545ac94de0c55dacdb4b3 2024-12-08T04:00:00,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/00af385ff45c4586bdfb2907f72b6d61 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/00af385ff45c4586bdfb2907f72b6d61 2024-12-08T04:00:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T04:00:00,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/a4b8715d4ec34198bb656f5a171e4632 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/a4b8715d4ec34198bb656f5a171e4632 2024-12-08T04:00:00,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/02c2559cd0024ec3849a79424a638e13 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/02c2559cd0024ec3849a79424a638e13 2024-12-08T04:00:00,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2d851ccf2b7148c898d356512ef66628 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/2d851ccf2b7148c898d356512ef66628 2024-12-08T04:00:00,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5ca89c09f6c94ec88126e4a5d6252d7b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5ca89c09f6c94ec88126e4a5d6252d7b 2024-12-08T04:00:00,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c203524766c34856b3b78f35cc55f258 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/c203524766c34856b3b78f35cc55f258 2024-12-08T04:00:00,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/37262e3c1e9541f1bfcfd0b22dbf968d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/37262e3c1e9541f1bfcfd0b22dbf968d 2024-12-08T04:00:00,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/4cbba6a0117847c3bf55f04be4501453 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/4cbba6a0117847c3bf55f04be4501453 2024-12-08T04:00:00,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/35928cf049ee4e6ebc7a5f07ce783c4e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/35928cf049ee4e6ebc7a5f07ce783c4e 2024-12-08T04:00:00,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8f4569f954944d5d9a1b3fdf167d8c31 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8f4569f954944d5d9a1b3fdf167d8c31 2024-12-08T04:00:00,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5cd4acff677a463e8f23cc7849dc0c37 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/5cd4acff677a463e8f23cc7849dc0c37 2024-12-08T04:00:00,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/88d44da6a6cd4b3e9e5bc1d135b5f602 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/88d44da6a6cd4b3e9e5bc1d135b5f602 2024-12-08T04:00:00,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/160eb164ae2b4cdd978db2f6471ce18f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/160eb164ae2b4cdd978db2f6471ce18f 2024-12-08T04:00:00,923 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/recovered.edits/705.seqid, newMaxSeqId=705, maxSeqId=1 2024-12-08T04:00:00,923 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. 2024-12-08T04:00:00,924 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for eaa9250e0e196d2f3604a9c3f7ea0f7e: Waiting for close lock at 1733630399933Running coprocessor pre-close hooks at 1733630399933Disabling compacts and flushes for region at 1733630399933Disabling writes for close at 1733630399933Obtaining lock to block concurrent updates at 1733630399933Preparing flush snapshotting stores in eaa9250e0e196d2f3604a9c3f7ea0f7e at 1733630399933Finished memstore snapshotting TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733630399933Flushing stores of TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e. at 1733630399934 (+1 ms)Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/A: creating writer at 1733630399934Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/A: appending metadata at 1733630399937 (+3 ms)Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/A: closing flushed file at 1733630399937Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/B: creating writer at 1733630399946 (+9 ms)Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/B: appending metadata at 1733630399948 (+2 ms)Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/B: closing flushed file at 1733630399948Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/C: creating writer at 1733630400373 (+425 ms)Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/C: appending metadata at 1733630400375 (+2 ms)Flushing eaa9250e0e196d2f3604a9c3f7ea0f7e/C: closing flushed file at 1733630400375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4446a37f: reopening flushed file at 1733630400783 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3517e01f: reopening flushed file at 1733630400788 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a7ed162: reopening flushed file at 1733630400793 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for eaa9250e0e196d2f3604a9c3f7ea0f7e in 866ms, sequenceid=702, compaction requested=true at 1733630400799 (+6 ms)Writing region close event to WAL at 1733630400920 (+121 ms)Running coprocessor post-close hooks at 1733630400923 (+3 ms)Closed at 1733630400923 2024-12-08T04:00:00,925 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T04:00:00,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=eaa9250e0e196d2f3604a9c3f7ea0f7e, regionState=CLOSED 2024-12-08T04:00:00,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:00:00,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-08T04:00:00,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure eaa9250e0e196d2f3604a9c3f7ea0f7e, server=32df4b4bb439,39919,1733630308479 in 1.1480 sec 2024-12-08T04:00:00,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-08T04:00:00,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eaa9250e0e196d2f3604a9c3f7ea0f7e, UNASSIGN in 1.1530 sec 2024-12-08T04:00:00,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-08T04:00:00,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1560 sec 2024-12-08T04:00:00,936 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630400935"}]},"ts":"1733630400935"} 2024-12-08T04:00:00,937 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T04:00:00,937 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T04:00:00,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-12-08T04:00:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T04:00:01,928 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:01,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T04:00:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:01,930 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-08T04:00:01,930 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:01,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T04:00:01,934 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/recovered.edits] 2024-12-08T04:00:01,936 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/63035356972442608dd47e69bbdb1b36 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/63035356972442608dd47e69bbdb1b36 2024-12-08T04:00:01,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7950a84d2ee2481ab2ec1cddc9a67fb3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/7950a84d2ee2481ab2ec1cddc9a67fb3 2024-12-08T04:00:01,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d23c0439091342e7a35ce2f71f7b7ef3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/d23c0439091342e7a35ce2f71f7b7ef3 2024-12-08T04:00:01,939 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/df01567b39c04fd78c1b2d21846429a4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/A/df01567b39c04fd78c1b2d21846429a4 2024-12-08T04:00:01,941 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/37b3c2bb464f48c1a905dacdc5c07531 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/37b3c2bb464f48c1a905dacdc5c07531 2024-12-08T04:00:01,942 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8dbba20d46694135b79d92c9e2bdf02a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/8dbba20d46694135b79d92c9e2bdf02a 2024-12-08T04:00:01,942 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/afed4c1de84945b1982f60eb5264ef34 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/afed4c1de84945b1982f60eb5264ef34 2024-12-08T04:00:01,943 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d16a9079ec574f2d9cf640cf2c41b2f7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/B/d16a9079ec574f2d9cf640cf2c41b2f7 2024-12-08T04:00:01,945 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/07a4ca9f787848928b8c9c63410db52f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/07a4ca9f787848928b8c9c63410db52f 2024-12-08T04:00:01,946 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/30b51394e0f74e3a97817a5299b52a32 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/30b51394e0f74e3a97817a5299b52a32 2024-12-08T04:00:01,947 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8568cea969e74c08be7ba7b038dea9e7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/8568cea969e74c08be7ba7b038dea9e7 2024-12-08T04:00:01,958 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb129e7b51fb4efda6a10cc42b491ead to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/C/fb129e7b51fb4efda6a10cc42b491ead 2024-12-08T04:00:01,961 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/recovered.edits/705.seqid to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e/recovered.edits/705.seqid 2024-12-08T04:00:01,962 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/eaa9250e0e196d2f3604a9c3f7ea0f7e 2024-12-08T04:00:01,962 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T04:00:01,964 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:01,968 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T04:00:01,970 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T04:00:01,971 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:01,971 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T04:00:01,971 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733630401971"}]},"ts":"9223372036854775807"} 2024-12-08T04:00:01,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T04:00:01,974 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eaa9250e0e196d2f3604a9c3f7ea0f7e, NAME => 'TestAcidGuarantees,,1733630370609.eaa9250e0e196d2f3604a9c3f7ea0f7e.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T04:00:01,974 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T04:00:01,974 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733630401974"}]},"ts":"9223372036854775807"} 2024-12-08T04:00:01,975 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T04:00:01,976 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:01,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-12-08T04:00:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-08T04:00:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T04:00:02,040 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:02,047 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 359) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=16269 (was 16567) 2024-12-08T04:00:02,055 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=16268 2024-12-08T04:00:02,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T04:00:02,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:00:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:02,058 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T04:00:02,058 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:02,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-08T04:00:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T04:00:02,059 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T04:00:02,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742155_1331 (size=1061) 2024-12-08T04:00:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T04:00:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T04:00:02,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T04:00:02,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742156_1332 (size=53) 2024-12-08T04:00:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T04:00:02,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:00:02,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00ee06bc7c2f5d203c30b1a99ea00704, disabling compactions & flushes 2024-12-08T04:00:02,875 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:02,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:02,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. after waiting 0 ms 2024-12-08T04:00:02,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:02,875 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:02,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00ee06bc7c2f5d203c30b1a99ea00704: Waiting for close lock at 1733630402875Disabling compacts and flushes for region at 1733630402875Disabling writes for close at 1733630402875Writing region close event to WAL at 1733630402875Closed at 1733630402875 2024-12-08T04:00:02,876 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T04:00:02,877 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733630402876"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630402876"}]},"ts":"1733630402876"} 2024-12-08T04:00:02,878 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T04:00:02,879 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T04:00:02,879 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630402879"}]},"ts":"1733630402879"} 2024-12-08T04:00:02,880 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T04:00:02,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, ASSIGN}] 2024-12-08T04:00:02,882 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, ASSIGN 2024-12-08T04:00:02,882 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, ASSIGN; state=OFFLINE, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=false 2024-12-08T04:00:03,033 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:03,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, ASSIGN because future has completed 2024-12-08T04:00:03,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:00:03,189 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:03,189 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:00:03,189 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,189 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:00:03,189 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,190 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,191 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,192 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:03,192 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ee06bc7c2f5d203c30b1a99ea00704 columnFamilyName A 2024-12-08T04:00:03,192 DEBUG [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:03,192 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(327): Store=00ee06bc7c2f5d203c30b1a99ea00704/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:03,192 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,193 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:03,193 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ee06bc7c2f5d203c30b1a99ea00704 columnFamilyName B 2024-12-08T04:00:03,193 DEBUG [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:03,193 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(327): Store=00ee06bc7c2f5d203c30b1a99ea00704/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:03,193 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,194 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:03,194 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ee06bc7c2f5d203c30b1a99ea00704 columnFamilyName C 2024-12-08T04:00:03,194 DEBUG [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:03,195 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(327): Store=00ee06bc7c2f5d203c30b1a99ea00704/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:03,195 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:03,195 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,195 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,196 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,196 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,196 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,197 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T04:00:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T04:00:03,198 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,199 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:00:03,200 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 00ee06bc7c2f5d203c30b1a99ea00704; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72163592, jitterRate=0.07532131671905518}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T04:00:03,200 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:03,200 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 00ee06bc7c2f5d203c30b1a99ea00704: Running coprocessor pre-open hook at 1733630403190Writing region info on filesystem at 1733630403190Initializing all the Stores at 1733630403190Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630403190Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630403190Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630403190Cleaning up temporary data from old regions at 1733630403196 (+6 ms)Running coprocessor post-open hooks at 1733630403200 (+4 ms)Region opened successfully at 1733630403200 2024-12-08T04:00:03,201 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., pid=93, masterSystemTime=1733630403186 2024-12-08T04:00:03,203 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:03,203 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:03,203 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:03,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:00:03,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-08T04:00:03,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 in 171 msec 2024-12-08T04:00:03,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-08T04:00:03,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, ASSIGN in 325 msec 2024-12-08T04:00:03,209 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T04:00:03,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630403209"}]},"ts":"1733630403209"} 2024-12-08T04:00:03,210 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T04:00:03,211 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T04:00:03,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-12-08T04:00:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-08T04:00:04,207 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:04,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:04,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:04,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:04,210 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15448242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:04,212 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f19dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:04,213 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:04,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:04,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T04:00:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T04:00:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b5a0c08 2024-12-08T04:00:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T04:00:04,219 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T04:00:04,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T04:00:04,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:00:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742157_1333 (size=1097) 2024-12-08T04:00:04,640 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T04:00:04,640 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T04:00:04,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T04:00:04,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, REOPEN/MOVE}] 2024-12-08T04:00:04,648 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, REOPEN/MOVE 2024-12-08T04:00:04,649 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:04,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, REOPEN/MOVE because future has completed 2024-12-08T04:00:04,651 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T04:00:04,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:00:04,804 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:04,804 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T04:00:04,804 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 00ee06bc7c2f5d203c30b1a99ea00704, disabling compactions & flushes 2024-12-08T04:00:04,804 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:04,804 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:04,804 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. after waiting 0 ms 2024-12-08T04:00:04,804 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:04,807 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T04:00:04,808 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:04,808 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 00ee06bc7c2f5d203c30b1a99ea00704: Waiting for close lock at 1733630404804Running coprocessor pre-close hooks at 1733630404804Disabling compacts and flushes for region at 1733630404804Disabling writes for close at 1733630404804Writing region close event to WAL at 1733630404805 (+1 ms)Running coprocessor post-close hooks at 1733630404808 (+3 ms)Closed at 1733630404808 2024-12-08T04:00:04,808 WARN [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 00ee06bc7c2f5d203c30b1a99ea00704 to self. 2024-12-08T04:00:04,810 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:04,810 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=CLOSED 2024-12-08T04:00:04,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:00:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-08T04:00:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 in 161 msec 2024-12-08T04:00:04,814 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=true 2024-12-08T04:00:04,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:04,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, REOPEN/MOVE because future has completed 2024-12-08T04:00:04,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:00:05,121 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,121 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:00:05,121 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,121 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:00:05,121 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,121 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,122 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,123 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:05,123 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ee06bc7c2f5d203c30b1a99ea00704 columnFamilyName A 2024-12-08T04:00:05,124 DEBUG [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:05,124 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(327): Store=00ee06bc7c2f5d203c30b1a99ea00704/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:05,125 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,125 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:05,125 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ee06bc7c2f5d203c30b1a99ea00704 columnFamilyName B 2024-12-08T04:00:05,125 DEBUG [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:05,126 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(327): Store=00ee06bc7c2f5d203c30b1a99ea00704/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:05,126 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,126 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:05,126 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ee06bc7c2f5d203c30b1a99ea00704 columnFamilyName C 2024-12-08T04:00:05,126 DEBUG [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:05,127 INFO [StoreOpener-00ee06bc7c2f5d203c30b1a99ea00704-1 {}] regionserver.HStore(327): Store=00ee06bc7c2f5d203c30b1a99ea00704/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:05,127 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,127 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,127 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,128 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,129 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,129 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,129 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T04:00:05,131 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,132 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 00ee06bc7c2f5d203c30b1a99ea00704; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63003989, jitterRate=-0.0611674040555954}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T04:00:05,132 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,132 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 00ee06bc7c2f5d203c30b1a99ea00704: Running coprocessor pre-open hook at 1733630405121Writing region info on filesystem at 1733630405121Initializing all the Stores at 1733630405122 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630405122Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630405122Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630405122Cleaning up temporary data from old regions at 1733630405129 (+7 ms)Running coprocessor post-open hooks at 1733630405132 (+3 ms)Region opened successfully at 1733630405132 2024-12-08T04:00:05,133 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., pid=98, masterSystemTime=1733630405118 2024-12-08T04:00:05,134 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,135 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,135 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=OPEN, openSeqNum=5, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:00:05,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-08T04:00:05,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 in 170 msec 2024-12-08T04:00:05,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-08T04:00:05,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, REOPEN/MOVE in 492 msec 2024-12-08T04:00:05,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-08T04:00:05,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-12-08T04:00:05,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 920 msec 2024-12-08T04:00:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-08T04:00:05,143 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:05,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,144 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1178e28f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b4a1c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242dcaae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5b134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ecafede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,152 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@495b0651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33044d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,155 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a01952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba3e3a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cceb127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2124c940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a210db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,166 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ddb753c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,168 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46571659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,170 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2968841b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,172 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9b59db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58fc2edf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:05,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:05,177 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bce5a3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:05,179 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5f36c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:05,182 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,182 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,183 DEBUG [Thread-1521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,183 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,184 DEBUG [Thread-1532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,184 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,184 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,185 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,185 DEBUG [Thread-1534 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,185 DEBUG [Thread-1534 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,187 DEBUG [Thread-1519 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,187 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,187 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-08T04:00:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T04:00:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:05,202 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:05,203 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:05,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:00:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:00:05,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:05,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:05,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:05,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:05,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:05,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:05,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083d131084e5c14b4cad2ae8ebc0d0ac90_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630405196/Put/seqid=0 2024-12-08T04:00:05,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742158_1334 (size=14594) 2024-12-08T04:00:05,249 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:05,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083d131084e5c14b4cad2ae8ebc0d0ac90_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083d131084e5c14b4cad2ae8ebc0d0ac90_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/8b4f63c41a3f47cc95734ce34f5af6d8, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:05,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/8b4f63c41a3f47cc95734ce34f5af6d8 is 175, key is test_row_0/A:col10/1733630405196/Put/seqid=0 2024-12-08T04:00:05,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630405243, max timestamp=1733630405254Num uniques 859; 2024-12-08T04:00:05,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:00:05,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630405243, max timestamp=1733630405254Num uniques 859; 2024-12-08T04:00:05,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:00:05,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630405243, max timestamp=1733630405254Num uniques 859; 2024-12-08T04:00:05,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:00:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742159_1335 (size=39549) 2024-12-08T04:00:05,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/8b4f63c41a3f47cc95734ce34f5af6d8 2024-12-08T04:00:05,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/e690811b0dbd41bfa579aef16a434fae is 50, key is test_row_0/B:col10/1733630405196/Put/seqid=0 2024-12-08T04:00:05,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630465289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630465291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630465292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742160_1336 (size=12001) 2024-12-08T04:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630465293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/e690811b0dbd41bfa579aef16a434fae 2024-12-08T04:00:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T04:00:05,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630465302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/f4f215492e0c4c04a453ab3c95fc5dc2 is 50, key is test_row_0/C:col10/1733630405196/Put/seqid=0 2024-12-08T04:00:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742161_1337 (size=12001) 2024-12-08T04:00:05,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/f4f215492e0c4c04a453ab3c95fc5dc2 2024-12-08T04:00:05,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/8b4f63c41a3f47cc95734ce34f5af6d8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8 2024-12-08T04:00:05,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T04:00:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8, entries=200, sequenceid=14, filesize=38.6 K 2024-12-08T04:00:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/e690811b0dbd41bfa579aef16a434fae as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/e690811b0dbd41bfa579aef16a434fae 2024-12-08T04:00:05,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/e690811b0dbd41bfa579aef16a434fae, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T04:00:05,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/f4f215492e0c4c04a453ab3c95fc5dc2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/f4f215492e0c4c04a453ab3c95fc5dc2 2024-12-08T04:00:05,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/f4f215492e0c4c04a453ab3c95fc5dc2, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T04:00:05,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00ee06bc7c2f5d203c30b1a99ea00704 in 179ms, sequenceid=14, compaction requested=false 2024-12-08T04:00:05,384 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T04:00:05,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630405276, max timestamp=1733630405423Num uniques 1018; 2024-12-08T04:00:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T04:00:05,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-12-08T04:00:05,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630405276, max timestamp=1733630405423Num uniques 1018; 2024-12-08T04:00:05,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T04:00:05,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:05,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T04:00:05,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:05,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:05,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:05,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:05,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120877e079390f3141e09264e5de1f4f1974_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630405424/Put/seqid=0 2024-12-08T04:00:05,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742162_1338 (size=19474) 2024-12-08T04:00:05,465 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:05,470 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120877e079390f3141e09264e5de1f4f1974_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120877e079390f3141e09264e5de1f4f1974_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:05,471 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/09db63100afe4b7f9e5ac28dad206b10, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:05,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/09db63100afe4b7f9e5ac28dad206b10 is 175, key is test_row_0/A:col10/1733630405424/Put/seqid=0 2024-12-08T04:00:05,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630465462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630465462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630465468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742163_1339 (size=56733) 2024-12-08T04:00:05,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/09db63100afe4b7f9e5ac28dad206b10 2024-12-08T04:00:05,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630465480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630465480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/6dbe0ec86f57406c9b6026ed471dd309 is 50, key is test_row_0/B:col10/1733630405424/Put/seqid=0 2024-12-08T04:00:05,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742164_1340 (size=12001) 2024-12-08T04:00:05,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T04:00:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T04:00:05,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630465587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630465588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630465588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630465597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630465597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T04:00:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630465797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630465797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630465807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T04:00:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:05,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630465817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630465817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T04:00:05,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/6dbe0ec86f57406c9b6026ed471dd309 2024-12-08T04:00:05,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/48b4cbd5ede446f9b2196d1496d31b9d is 50, key is test_row_0/C:col10/1733630405424/Put/seqid=0 2024-12-08T04:00:05,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742165_1341 (size=12001) 2024-12-08T04:00:05,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/48b4cbd5ede446f9b2196d1496d31b9d 2024-12-08T04:00:05,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/09db63100afe4b7f9e5ac28dad206b10 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10 2024-12-08T04:00:05,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10, entries=300, sequenceid=54, filesize=55.4 K 2024-12-08T04:00:05,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/6dbe0ec86f57406c9b6026ed471dd309 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6dbe0ec86f57406c9b6026ed471dd309 2024-12-08T04:00:05,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6dbe0ec86f57406c9b6026ed471dd309, entries=150, sequenceid=54, filesize=11.7 K 2024-12-08T04:00:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/48b4cbd5ede446f9b2196d1496d31b9d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/48b4cbd5ede446f9b2196d1496d31b9d 2024-12-08T04:00:05,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/48b4cbd5ede446f9b2196d1496d31b9d, entries=150, sequenceid=54, filesize=11.7 K 2024-12-08T04:00:05,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~218.59 KB/223840, currentSize=100.63 KB/103050 for 00ee06bc7c2f5d203c30b1a99ea00704 in 541ms, sequenceid=54, compaction requested=false 2024-12-08T04:00:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:05,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:05,970 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ee88318945dc4ffa8be8d725097f9446_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630405464/Put/seqid=0 2024-12-08T04:00:05,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742166_1342 (size=12154) 2024-12-08T04:00:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:06,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:06,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630466127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630466134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630466136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630466137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630466140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630466247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630466247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630466247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630466257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T04:00:06,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:06,393 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ee88318945dc4ffa8be8d725097f9446_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ee88318945dc4ffa8be8d725097f9446_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c724471424d84127923be25bf5361f49, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:06,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c724471424d84127923be25bf5361f49 is 175, key is test_row_0/A:col10/1733630405464/Put/seqid=0 2024-12-08T04:00:06,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742167_1343 (size=30955) 2024-12-08T04:00:06,461 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T04:00:06,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630466457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630466457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630466457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630466467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630466657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630466767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630466767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630466767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630466777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:06,830 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=72, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c724471424d84127923be25bf5361f49 2024-12-08T04:00:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/689047e763944f8dbb925451578ec5bf is 50, key is test_row_0/B:col10/1733630405464/Put/seqid=0 2024-12-08T04:00:06,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742168_1344 (size=12001) 2024-12-08T04:00:06,856 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/689047e763944f8dbb925451578ec5bf 2024-12-08T04:00:06,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/71082c17817b421d95ce664c0f76ecf7 is 50, key is test_row_0/C:col10/1733630405464/Put/seqid=0 2024-12-08T04:00:06,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742169_1345 (size=12001) 2024-12-08T04:00:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630467277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:07,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630467277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630467287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:07,292 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/71082c17817b421d95ce664c0f76ecf7 2024-12-08T04:00:07,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c724471424d84127923be25bf5361f49 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49 2024-12-08T04:00:07,300 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49, entries=150, sequenceid=72, filesize=30.2 K 2024-12-08T04:00:07,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/689047e763944f8dbb925451578ec5bf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/689047e763944f8dbb925451578ec5bf 2024-12-08T04:00:07,303 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/689047e763944f8dbb925451578ec5bf, entries=150, sequenceid=72, filesize=11.7 K 2024-12-08T04:00:07,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630467297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:07,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/71082c17817b421d95ce664c0f76ecf7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/71082c17817b421d95ce664c0f76ecf7 2024-12-08T04:00:07,308 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/71082c17817b421d95ce664c0f76ecf7, entries=150, sequenceid=72, filesize=11.7 K 2024-12-08T04:00:07,309 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1339ms, sequenceid=72, compaction requested=true 2024-12-08T04:00:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:07,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-08T04:00:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-08T04:00:07,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-08T04:00:07,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-12-08T04:00:07,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1160 sec 2024-12-08T04:00:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T04:00:07,347 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:07,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-08T04:00:07,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T04:00:07,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:07,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T04:00:07,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:07,504 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:07,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081c971a7cd4cc4849a8ed53bffdcf76e9_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630406137/Put/seqid=0 2024-12-08T04:00:07,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742170_1346 (size=12154) 2024-12-08T04:00:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T04:00:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:07,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:07,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630467840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:07,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:07,927 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081c971a7cd4cc4849a8ed53bffdcf76e9_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081c971a7cd4cc4849a8ed53bffdcf76e9_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:07,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/df093072fefe49349247aa61306b5a9c, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:07,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/df093072fefe49349247aa61306b5a9c is 175, key is test_row_0/A:col10/1733630406137/Put/seqid=0 2024-12-08T04:00:07,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742171_1347 (size=30955) 2024-12-08T04:00:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630467957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T04:00:08,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630468167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:08,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630468287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,290 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630468297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630468297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,300 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:08,301 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:08,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630468317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,323 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:08,338 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/df093072fefe49349247aa61306b5a9c 2024-12-08T04:00:08,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ecd92e54892e44e2aea07ce93e778384 is 50, key is test_row_0/B:col10/1733630406137/Put/seqid=0 2024-12-08T04:00:08,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742172_1348 (size=12001) 2024-12-08T04:00:08,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630468477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T04:00:08,763 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ecd92e54892e44e2aea07ce93e778384 2024-12-08T04:00:08,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/605f1d2b93884517a2135e49a32e3e37 is 50, key is test_row_0/C:col10/1733630406137/Put/seqid=0 2024-12-08T04:00:08,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742173_1349 (size=12001) 2024-12-08T04:00:08,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630468987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:09,172 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/605f1d2b93884517a2135e49a32e3e37 2024-12-08T04:00:09,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/df093072fefe49349247aa61306b5a9c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c 2024-12-08T04:00:09,181 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c, entries=150, sequenceid=86, filesize=30.2 K 2024-12-08T04:00:09,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ecd92e54892e44e2aea07ce93e778384 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ecd92e54892e44e2aea07ce93e778384 2024-12-08T04:00:09,186 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ecd92e54892e44e2aea07ce93e778384, entries=150, sequenceid=86, filesize=11.7 K 2024-12-08T04:00:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/605f1d2b93884517a2135e49a32e3e37 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/605f1d2b93884517a2135e49a32e3e37 2024-12-08T04:00:09,190 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/605f1d2b93884517a2135e49a32e3e37, entries=150, sequenceid=86, filesize=11.7 K 2024-12-08T04:00:09,191 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1686ms, sequenceid=86, compaction requested=true 2024-12-08T04:00:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-08T04:00:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-08T04:00:09,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-08T04:00:09,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-12-08T04:00:09,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8470 sec 2024-12-08T04:00:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T04:00:09,497 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:09,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-08T04:00:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T04:00:09,499 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:09,499 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:09,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T04:00:09,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:09,652 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d835daba22564410970c56a5173094f3_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630407776/Put/seqid=0 2024-12-08T04:00:09,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742174_1350 (size=12154) 2024-12-08T04:00:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T04:00:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:10,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:10,069 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d835daba22564410970c56a5173094f3_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d835daba22564410970c56a5173094f3_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e72d240f1c2c42fe83981902ebbc3cc3, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:10,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e72d240f1c2c42fe83981902ebbc3cc3 is 175, key is test_row_0/A:col10/1733630407776/Put/seqid=0 2024-12-08T04:00:10,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742175_1351 (size=30955) 2024-12-08T04:00:10,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630470093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T04:00:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630470207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630470317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630470317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630470317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,321 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:10,321 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,322 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630470347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,357 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630470417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,477 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e72d240f1c2c42fe83981902ebbc3cc3 2024-12-08T04:00:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/347aa37ec59c45a884e8dd52999e24d8 is 50, key is test_row_0/B:col10/1733630407776/Put/seqid=0 2024-12-08T04:00:10,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742176_1352 (size=12001) 2024-12-08T04:00:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T04:00:10,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630470727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:10,905 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/347aa37ec59c45a884e8dd52999e24d8 2024-12-08T04:00:10,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/687d676070644ebda8034da6c693264a is 50, key is test_row_0/C:col10/1733630407776/Put/seqid=0 2024-12-08T04:00:10,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742177_1353 (size=12001) 2024-12-08T04:00:11,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630471237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:11,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:11,314 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/687d676070644ebda8034da6c693264a 2024-12-08T04:00:11,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e72d240f1c2c42fe83981902ebbc3cc3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3 2024-12-08T04:00:11,321 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3, entries=150, sequenceid=104, filesize=30.2 K 2024-12-08T04:00:11,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/347aa37ec59c45a884e8dd52999e24d8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/347aa37ec59c45a884e8dd52999e24d8 2024-12-08T04:00:11,325 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/347aa37ec59c45a884e8dd52999e24d8, entries=150, sequenceid=104, filesize=11.7 K 2024-12-08T04:00:11,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/687d676070644ebda8034da6c693264a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/687d676070644ebda8034da6c693264a 2024-12-08T04:00:11,328 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/687d676070644ebda8034da6c693264a, entries=150, sequenceid=104, filesize=11.7 K 2024-12-08T04:00:11,329 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1676ms, sequenceid=104, compaction requested=true 2024-12-08T04:00:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-08T04:00:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-08T04:00:11,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-08T04:00:11,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-12-08T04:00:11,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8340 sec 2024-12-08T04:00:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T04:00:11,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:11,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-08T04:00:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T04:00:11,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:11,651 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:11,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T04:00:11,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T04:00:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:11,804 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T04:00:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:11,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f899ea6c55984f84b05d4024bd17cce2_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630410081/Put/seqid=0 2024-12-08T04:00:11,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742178_1354 (size=12154) 2024-12-08T04:00:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T04:00:12,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:12,216 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f899ea6c55984f84b05d4024bd17cce2_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f899ea6c55984f84b05d4024bd17cce2_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:12,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/2f29c57301104ede8317f9d5ef5f7d04, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:12,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/2f29c57301104ede8317f9d5ef5f7d04 is 175, key is test_row_0/A:col10/1733630410081/Put/seqid=0 2024-12-08T04:00:12,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742179_1355 (size=30955) 2024-12-08T04:00:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:12,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T04:00:12,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630472365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:12,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630472477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:12,625 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/2f29c57301104ede8317f9d5ef5f7d04 2024-12-08T04:00:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/0e091c493297415ca8a8f3cb06be2538 is 50, key is test_row_0/B:col10/1733630410081/Put/seqid=0 2024-12-08T04:00:12,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742180_1356 (size=12001) 2024-12-08T04:00:12,641 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/0e091c493297415ca8a8f3cb06be2538 2024-12-08T04:00:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/feddd737704942eab275e42918c39441 is 50, key is test_row_0/C:col10/1733630410081/Put/seqid=0 2024-12-08T04:00:12,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742181_1357 (size=12001) 2024-12-08T04:00:12,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630472687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:12,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T04:00:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630472997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:13,058 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/feddd737704942eab275e42918c39441 2024-12-08T04:00:13,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/2f29c57301104ede8317f9d5ef5f7d04 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04 2024-12-08T04:00:13,065 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04, entries=150, sequenceid=118, filesize=30.2 K 2024-12-08T04:00:13,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/0e091c493297415ca8a8f3cb06be2538 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0e091c493297415ca8a8f3cb06be2538 2024-12-08T04:00:13,069 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0e091c493297415ca8a8f3cb06be2538, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T04:00:13,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/feddd737704942eab275e42918c39441 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/feddd737704942eab275e42918c39441 2024-12-08T04:00:13,073 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/feddd737704942eab275e42918c39441, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T04:00:13,074 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1270ms, sequenceid=118, compaction requested=true 2024-12-08T04:00:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-08T04:00:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-08T04:00:13,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-08T04:00:13,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-12-08T04:00:13,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4320 sec 2024-12-08T04:00:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:13,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T04:00:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:13,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f8d862040e8f48f9bb3c9f71d856b204_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:13,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742182_1358 (size=14794) 2024-12-08T04:00:13,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:13,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f8d862040e8f48f9bb3c9f71d856b204_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f8d862040e8f48f9bb3c9f71d856b204_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:13,558 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c957194732f64b1aa75e20268746e845, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:13,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c957194732f64b1aa75e20268746e845 is 175, key is test_row_0/A:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:13,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742183_1359 (size=39749) 2024-12-08T04:00:13,570 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c957194732f64b1aa75e20268746e845 2024-12-08T04:00:13,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ba91961a88eb4870a5104649bfc6431e is 50, key is test_row_0/B:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:13,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742184_1360 (size=12151) 2024-12-08T04:00:13,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ba91961a88eb4870a5104649bfc6431e 2024-12-08T04:00:13,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/0410697a4dc94b2e966b5d50a9d2b43d is 50, key is test_row_0/C:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:13,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742185_1361 (size=12151) 2024-12-08T04:00:13,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630473684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T04:00:13,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:13,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-08T04:00:13,800 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:13,800 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:13,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630473797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T04:00:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T04:00:13,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T04:00:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:13,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:13,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:13,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:13,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:14,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630474017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/0410697a4dc94b2e966b5d50a9d2b43d 2024-12-08T04:00:14,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/c957194732f64b1aa75e20268746e845 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845 2024-12-08T04:00:14,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845, entries=200, sequenceid=137, filesize=38.8 K 2024-12-08T04:00:14,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ba91961a88eb4870a5104649bfc6431e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ba91961a88eb4870a5104649bfc6431e 2024-12-08T04:00:14,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ba91961a88eb4870a5104649bfc6431e, entries=150, sequenceid=137, filesize=11.9 K 2024-12-08T04:00:14,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/0410697a4dc94b2e966b5d50a9d2b43d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0410697a4dc94b2e966b5d50a9d2b43d 2024-12-08T04:00:14,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0410697a4dc94b2e966b5d50a9d2b43d, entries=150, sequenceid=137, filesize=11.9 K 2024-12-08T04:00:14,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 00ee06bc7c2f5d203c30b1a99ea00704 in 550ms, sequenceid=137, compaction requested=true 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:14,079 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T04:00:14,080 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-08T04:00:14,084 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 259851 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-08T04:00:14,084 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/A is initiating minor compaction (all files) 2024-12-08T04:00:14,084 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/A in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:14,084 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=253.8 K 2024-12-08T04:00:14,084 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:14,084 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845] 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b4f63c41a3f47cc95734ce34f5af6d8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733630405196 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84157 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/B is initiating minor compaction (all files) 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09db63100afe4b7f9e5ac28dad206b10, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733630405276 2024-12-08T04:00:14,085 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/B in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:14,085 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/e690811b0dbd41bfa579aef16a434fae, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6dbe0ec86f57406c9b6026ed471dd309, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/689047e763944f8dbb925451578ec5bf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ecd92e54892e44e2aea07ce93e778384, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/347aa37ec59c45a884e8dd52999e24d8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0e091c493297415ca8a8f3cb06be2538, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ba91961a88eb4870a5104649bfc6431e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=82.2 K 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c724471424d84127923be25bf5361f49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1733630405464 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e690811b0dbd41bfa579aef16a434fae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733630405196 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting df093072fefe49349247aa61306b5a9c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733630406121 2024-12-08T04:00:14,085 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dbe0ec86f57406c9b6026ed471dd309, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733630405286 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e72d240f1c2c42fe83981902ebbc3cc3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733630407776 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 689047e763944f8dbb925451578ec5bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1733630405464 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f29c57301104ede8317f9d5ef5f7d04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733630410042 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd92e54892e44e2aea07ce93e778384, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733630406121 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 347aa37ec59c45a884e8dd52999e24d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733630407776 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting c957194732f64b1aa75e20268746e845, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733630412306 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e091c493297415ca8a8f3cb06be2538, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733630410042 2024-12-08T04:00:14,086 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ba91961a88eb4870a5104649bfc6431e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733630412306 2024-12-08T04:00:14,095 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:14,099 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#B#compaction#309 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:14,099 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/6190ccfe29bc4575b2472eef4bf46719 is 50, key is test_row_0/B:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:14,100 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120888fd436262e84a4f905798c203e1c61e_00ee06bc7c2f5d203c30b1a99ea00704 store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:14,105 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120888fd436262e84a4f905798c203e1c61e_00ee06bc7c2f5d203c30b1a99ea00704, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:14,105 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120888fd436262e84a4f905798c203e1c61e_00ee06bc7c2f5d203c30b1a99ea00704 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:14,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T04:00:14,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:14,107 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T04:00:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:14,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:14,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742186_1362 (size=12391) 2024-12-08T04:00:14,113 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/6190ccfe29bc4575b2472eef4bf46719 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6190ccfe29bc4575b2472eef4bf46719 2024-12-08T04:00:14,117 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/B of 00ee06bc7c2f5d203c30b1a99ea00704 into 6190ccfe29bc4575b2472eef4bf46719(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:14,117 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:14,117 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/B, priority=9, startTime=1733630414079; duration=0sec 2024-12-08T04:00:14,117 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:14,117 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:B 2024-12-08T04:00:14,117 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-12-08T04:00:14,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742187_1363 (size=4469) 2024-12-08T04:00:14,118 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#A#compaction#308 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:14,119 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/776baeec5c8e4a5287c34ef1c04093db is 175, key is test_row_0/A:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e6d6dcd3398a48ac90524115b30bc0dc_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630413643/Put/seqid=0 2024-12-08T04:00:14,122 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84157 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-12-08T04:00:14,122 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/C is initiating minor compaction (all files) 2024-12-08T04:00:14,122 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/C in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:14,122 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/f4f215492e0c4c04a453ab3c95fc5dc2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/48b4cbd5ede446f9b2196d1496d31b9d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/71082c17817b421d95ce664c0f76ecf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/605f1d2b93884517a2135e49a32e3e37, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/687d676070644ebda8034da6c693264a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/feddd737704942eab275e42918c39441, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0410697a4dc94b2e966b5d50a9d2b43d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=82.2 K 2024-12-08T04:00:14,123 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f215492e0c4c04a453ab3c95fc5dc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733630405196 2024-12-08T04:00:14,123 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b4cbd5ede446f9b2196d1496d31b9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733630405286 2024-12-08T04:00:14,123 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 71082c17817b421d95ce664c0f76ecf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=72, earliestPutTs=1733630405464 2024-12-08T04:00:14,123 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 605f1d2b93884517a2135e49a32e3e37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733630406121 2024-12-08T04:00:14,124 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 687d676070644ebda8034da6c693264a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733630407776 2024-12-08T04:00:14,124 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting feddd737704942eab275e42918c39441, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733630410042 2024-12-08T04:00:14,124 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0410697a4dc94b2e966b5d50a9d2b43d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733630412306 2024-12-08T04:00:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T04:00:14,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742189_1365 (size=12304) 2024-12-08T04:00:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:14,134 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e6d6dcd3398a48ac90524115b30bc0dc_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e6d6dcd3398a48ac90524115b30bc0dc_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:14,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/da96cd37d4bf431288f8209329070193, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/da96cd37d4bf431288f8209329070193 is 175, key is test_row_0/A:col10/1733630413643/Put/seqid=0 2024-12-08T04:00:14,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742188_1364 (size=31345) 2024-12-08T04:00:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742190_1366 (size=31105) 2024-12-08T04:00:14,143 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#C#compaction#311 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:14,144 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/5e3c94c77d1443dfb2aa0f4200fffc82 is 50, key is test_row_0/C:col10/1733630412364/Put/seqid=0 2024-12-08T04:00:14,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742191_1367 (size=12391) 2024-12-08T04:00:14,151 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/776baeec5c8e4a5287c34ef1c04093db as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/776baeec5c8e4a5287c34ef1c04093db 2024-12-08T04:00:14,154 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/5e3c94c77d1443dfb2aa0f4200fffc82 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/5e3c94c77d1443dfb2aa0f4200fffc82 2024-12-08T04:00:14,156 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/A of 00ee06bc7c2f5d203c30b1a99ea00704 into 776baeec5c8e4a5287c34ef1c04093db(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:14,156 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:14,156 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/A, priority=9, startTime=1733630414079; duration=0sec 2024-12-08T04:00:14,156 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:14,156 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:A 2024-12-08T04:00:14,159 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/C of 00ee06bc7c2f5d203c30b1a99ea00704 into 5e3c94c77d1443dfb2aa0f4200fffc82(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:14,159 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:14,159 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/C, priority=9, startTime=1733630414079; duration=0sec 2024-12-08T04:00:14,159 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:14,159 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:C 2024-12-08T04:00:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:14,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630474355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630474358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630474358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630474365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630474397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,402 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T04:00:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630474467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630474477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630474477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630474477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,541 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/da96cd37d4bf431288f8209329070193 2024-12-08T04:00:14,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/dc5531c4942b405c9187dcd4c2f7e537 is 50, key is test_row_0/B:col10/1733630413643/Put/seqid=0 2024-12-08T04:00:14,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742192_1368 (size=12151) 2024-12-08T04:00:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630474677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630474687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630474688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630474688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T04:00:14,952 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/dc5531c4942b405c9187dcd4c2f7e537 2024-12-08T04:00:14,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1bd6d00664ab4d239bff95544835410a is 50, key is test_row_0/C:col10/1733630413643/Put/seqid=0 2024-12-08T04:00:14,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742193_1369 (size=12151) 2024-12-08T04:00:14,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630474987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630474997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630474997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630474997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,362 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1bd6d00664ab4d239bff95544835410a 2024-12-08T04:00:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/da96cd37d4bf431288f8209329070193 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193 2024-12-08T04:00:15,368 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193, entries=150, sequenceid=150, filesize=30.4 K 2024-12-08T04:00:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/dc5531c4942b405c9187dcd4c2f7e537 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/dc5531c4942b405c9187dcd4c2f7e537 2024-12-08T04:00:15,372 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/dc5531c4942b405c9187dcd4c2f7e537, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T04:00:15,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1bd6d00664ab4d239bff95544835410a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1bd6d00664ab4d239bff95544835410a 2024-12-08T04:00:15,376 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1bd6d00664ab4d239bff95544835410a, entries=150, sequenceid=150, filesize=11.9 K 2024-12-08T04:00:15,377 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1270ms, sequenceid=150, compaction requested=false 2024-12-08T04:00:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-08T04:00:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-08T04:00:15,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-08T04:00:15,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-12-08T04:00:15,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5810 sec 2024-12-08T04:00:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:15,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T04:00:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:15,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208055cab27a4734c43a09f55a86e518259_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:15,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742194_1370 (size=14794) 2024-12-08T04:00:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630475557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630475558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630475558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630475562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630475678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630475678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630475678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630475678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630475888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630475897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630475897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630475897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:15,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:15,937 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208055cab27a4734c43a09f55a86e518259_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208055cab27a4734c43a09f55a86e518259_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:15,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/72e2fa7614b54bf4bba73ad135ec622b, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:15,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/72e2fa7614b54bf4bba73ad135ec622b is 175, key is test_row_0/A:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T04:00:15,957 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:15,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-08T04:00:15,960 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:15,960 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:15,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:15,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742195_1371 (size=39749) 2024-12-08T04:00:15,988 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/72e2fa7614b54bf4bba73ad135ec622b 2024-12-08T04:00:16,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/1159f92fdab544b991e9b0cee0f32913 is 50, key is test_row_0/B:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:16,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742196_1372 (size=12151) 2024-12-08T04:00:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:16,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630476197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630476207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630476217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630476217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:16,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/1159f92fdab544b991e9b0cee0f32913 2024-12-08T04:00:16,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/651d210c0f4048c791f7a798bd352ded is 50, key is test_row_0/C:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:16,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742197_1373 (size=12151) 2024-12-08T04:00:16,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:16,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:16,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:16,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630476707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630476717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630476727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630476727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:16,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/651d210c0f4048c791f7a798bd352ded 2024-12-08T04:00:16,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/72e2fa7614b54bf4bba73ad135ec622b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b 2024-12-08T04:00:16,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b, entries=200, sequenceid=173, filesize=38.8 K 2024-12-08T04:00:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/1159f92fdab544b991e9b0cee0f32913 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1159f92fdab544b991e9b0cee0f32913 2024-12-08T04:00:16,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:16,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:16,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:16,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1159f92fdab544b991e9b0cee0f32913, entries=150, sequenceid=173, filesize=11.9 K 2024-12-08T04:00:16,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/651d210c0f4048c791f7a798bd352ded as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/651d210c0f4048c791f7a798bd352ded 2024-12-08T04:00:16,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/651d210c0f4048c791f7a798bd352ded, entries=150, sequenceid=173, filesize=11.9 K 2024-12-08T04:00:16,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1389ms, sequenceid=173, compaction requested=true 2024-12-08T04:00:16,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:16,902 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:16,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:16,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:16,902 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:16,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:16,905 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:16,905 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/B is initiating minor compaction (all files) 2024-12-08T04:00:16,905 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/B in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,905 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6190ccfe29bc4575b2472eef4bf46719, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/dc5531c4942b405c9187dcd4c2f7e537, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1159f92fdab544b991e9b0cee0f32913] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=35.8 K 2024-12-08T04:00:16,905 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:16,905 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/A is initiating minor compaction (all files) 2024-12-08T04:00:16,906 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/A in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,906 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/776baeec5c8e4a5287c34ef1c04093db, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=99.8 K 2024-12-08T04:00:16,906 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:16,906 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/776baeec5c8e4a5287c34ef1c04093db, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b] 2024-12-08T04:00:16,906 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776baeec5c8e4a5287c34ef1c04093db, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733630412306 2024-12-08T04:00:16,906 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6190ccfe29bc4575b2472eef4bf46719, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733630412306 2024-12-08T04:00:16,907 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting da96cd37d4bf431288f8209329070193, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733630413643 2024-12-08T04:00:16,907 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting dc5531c4942b405c9187dcd4c2f7e537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733630413643 2024-12-08T04:00:16,907 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e2fa7614b54bf4bba73ad135ec622b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733630414356 2024-12-08T04:00:16,907 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1159f92fdab544b991e9b0cee0f32913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733630414356 2024-12-08T04:00:16,937 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#B#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:16,937 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:16,937 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/83303d17659a4e5db0d8f9eae30dd5e2 is 50, key is test_row_0/B:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:16,948 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208f46b78e356054e2ca937ab1ba2375a17_00ee06bc7c2f5d203c30b1a99ea00704 store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:16,950 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208f46b78e356054e2ca937ab1ba2375a17_00ee06bc7c2f5d203c30b1a99ea00704, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:16,950 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f46b78e356054e2ca937ab1ba2375a17_00ee06bc7c2f5d203c30b1a99ea00704 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:16,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742199_1375 (size=4469) 2024-12-08T04:00:16,969 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#A#compaction#318 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:16,969 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/5189ce0ee22f4051b25775f81810d303 is 175, key is test_row_0/A:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:16,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742198_1374 (size=12493) 2024-12-08T04:00:16,988 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/83303d17659a4e5db0d8f9eae30dd5e2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/83303d17659a4e5db0d8f9eae30dd5e2 2024-12-08T04:00:16,997 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/B of 00ee06bc7c2f5d203c30b1a99ea00704 into 83303d17659a4e5db0d8f9eae30dd5e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:16,997 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:16,997 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/B, priority=13, startTime=1733630416902; duration=0sec 2024-12-08T04:00:16,997 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:16,997 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:B 2024-12-08T04:00:16,997 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:16,999 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:17,000 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/C is initiating minor compaction (all files) 2024-12-08T04:00:17,000 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/C in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:17,001 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/5e3c94c77d1443dfb2aa0f4200fffc82, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1bd6d00664ab4d239bff95544835410a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/651d210c0f4048c791f7a798bd352ded] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=35.8 K 2024-12-08T04:00:17,002 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e3c94c77d1443dfb2aa0f4200fffc82, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733630412306 2024-12-08T04:00:17,002 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd6d00664ab4d239bff95544835410a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733630413643 2024-12-08T04:00:17,003 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 651d210c0f4048c791f7a798bd352ded, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733630414356 2024-12-08T04:00:17,013 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#C#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:17,014 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/88ea0822295844bd8d46a52a4c0dd05f is 50, key is test_row_0/C:col10/1733630414356/Put/seqid=0 2024-12-08T04:00:17,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742200_1376 (size=31447) 2024-12-08T04:00:17,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-08T04:00:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:17,031 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T04:00:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:17,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742201_1377 (size=12493) 2024-12-08T04:00:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120853ba8783faad49489cc83bf7e7ead74b_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630415557/Put/seqid=0 2024-12-08T04:00:17,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742202_1378 (size=12304) 2024-12-08T04:00:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:17,432 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/5189ce0ee22f4051b25775f81810d303 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5189ce0ee22f4051b25775f81810d303 2024-12-08T04:00:17,436 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/A of 00ee06bc7c2f5d203c30b1a99ea00704 into 5189ce0ee22f4051b25775f81810d303(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:17,436 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:17,436 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/A, priority=13, startTime=1733630416902; duration=0sec 2024-12-08T04:00:17,436 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:17,437 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:A 2024-12-08T04:00:17,440 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/88ea0822295844bd8d46a52a4c0dd05f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/88ea0822295844bd8d46a52a4c0dd05f 2024-12-08T04:00:17,443 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/C of 00ee06bc7c2f5d203c30b1a99ea00704 into 88ea0822295844bd8d46a52a4c0dd05f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:17,444 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:17,444 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/C, priority=13, startTime=1733630416903; duration=0sec 2024-12-08T04:00:17,444 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:17,444 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:C 2024-12-08T04:00:17,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:17,454 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120853ba8783faad49489cc83bf7e7ead74b_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853ba8783faad49489cc83bf7e7ead74b_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/aafe75ab80b54d72a854e96a98260b32, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:17,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/aafe75ab80b54d72a854e96a98260b32 is 175, key is test_row_0/A:col10/1733630415557/Put/seqid=0 2024-12-08T04:00:17,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742203_1379 (size=31105) 2024-12-08T04:00:17,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:17,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630477757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630477760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630477761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630477767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,859 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/aafe75ab80b54d72a854e96a98260b32 2024-12-08T04:00:17,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/8136126956c74eacad237937c94a8432 is 50, key is test_row_0/B:col10/1733630415557/Put/seqid=0 2024-12-08T04:00:17,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742204_1380 (size=12151) 2024-12-08T04:00:17,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630477877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630477877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630477877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630477877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630478087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630478087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630478088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630478088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:18,268 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/8136126956c74eacad237937c94a8432 2024-12-08T04:00:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1478453a2c6744ee9ad228a7547406b5 is 50, key is test_row_0/C:col10/1733630415557/Put/seqid=0 2024-12-08T04:00:18,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742205_1381 (size=12151) 2024-12-08T04:00:18,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630478408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630478408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630478408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630478408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,685 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1478453a2c6744ee9ad228a7547406b5 2024-12-08T04:00:18,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/aafe75ab80b54d72a854e96a98260b32 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32 2024-12-08T04:00:18,698 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32, entries=150, sequenceid=186, filesize=30.4 K 2024-12-08T04:00:18,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/8136126956c74eacad237937c94a8432 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8136126956c74eacad237937c94a8432 2024-12-08T04:00:18,703 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8136126956c74eacad237937c94a8432, entries=150, sequenceid=186, filesize=11.9 K 2024-12-08T04:00:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1478453a2c6744ee9ad228a7547406b5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1478453a2c6744ee9ad228a7547406b5 2024-12-08T04:00:18,716 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1478453a2c6744ee9ad228a7547406b5, entries=150, sequenceid=186, filesize=11.9 K 2024-12-08T04:00:18,717 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1686ms, sequenceid=186, compaction requested=false 2024-12-08T04:00:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-08T04:00:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-08T04:00:18,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-08T04:00:18,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7590 sec 2024-12-08T04:00:18,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.7650 sec 2024-12-08T04:00:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:18,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T04:00:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:18,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:18,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:18,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:18,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:18,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d17c454ba43540f4966e9bc944f61b31_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:18,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742206_1382 (size=12304) 2024-12-08T04:00:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630478934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630478935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630478936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630478941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630479047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630479057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630479057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630479057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630479267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630479267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630479267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630479267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:19,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d17c454ba43540f4966e9bc944f61b31_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d17c454ba43540f4966e9bc944f61b31_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:19,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/5e3bc7d1ba3447828eb566bf29385ae8, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:19,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/5e3bc7d1ba3447828eb566bf29385ae8 is 175, key is test_row_0/A:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:19,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742207_1383 (size=31105) 2024-12-08T04:00:19,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/5e3bc7d1ba3447828eb566bf29385ae8 2024-12-08T04:00:19,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ea0f4f2728774414ae8083c2a4f8a291 is 50, key is test_row_0/B:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:19,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742208_1384 (size=12151) 2024-12-08T04:00:19,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ea0f4f2728774414ae8083c2a4f8a291 2024-12-08T04:00:19,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/85084ad64bbb4c9a9443ddd1bbf39fc2 is 50, key is test_row_0/C:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:19,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742209_1385 (size=12151) 2024-12-08T04:00:19,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/85084ad64bbb4c9a9443ddd1bbf39fc2 2024-12-08T04:00:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/5e3bc7d1ba3447828eb566bf29385ae8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8 2024-12-08T04:00:19,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8, entries=150, sequenceid=209, filesize=30.4 K 2024-12-08T04:00:19,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/ea0f4f2728774414ae8083c2a4f8a291 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ea0f4f2728774414ae8083c2a4f8a291 2024-12-08T04:00:19,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ea0f4f2728774414ae8083c2a4f8a291, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T04:00:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/85084ad64bbb4c9a9443ddd1bbf39fc2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/85084ad64bbb4c9a9443ddd1bbf39fc2 2024-12-08T04:00:19,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/85084ad64bbb4c9a9443ddd1bbf39fc2, entries=150, sequenceid=209, filesize=11.9 K 2024-12-08T04:00:19,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 00ee06bc7c2f5d203c30b1a99ea00704 in 479ms, sequenceid=209, compaction requested=true 2024-12-08T04:00:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:19,403 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:19,403 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:19,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:19,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:19,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/B is initiating minor compaction (all files) 2024-12-08T04:00:19,404 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/B in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:19,404 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/83303d17659a4e5db0d8f9eae30dd5e2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8136126956c74eacad237937c94a8432, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ea0f4f2728774414ae8083c2a4f8a291] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=35.9 K 2024-12-08T04:00:19,404 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:19,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 83303d17659a4e5db0d8f9eae30dd5e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733630414356 2024-12-08T04:00:19,405 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/A is initiating minor compaction (all files) 2024-12-08T04:00:19,405 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/A in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:19,405 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5189ce0ee22f4051b25775f81810d303, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=91.5 K 2024-12-08T04:00:19,405 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:19,405 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5189ce0ee22f4051b25775f81810d303, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8] 2024-12-08T04:00:19,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8136126956c74eacad237937c94a8432, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733630415557 2024-12-08T04:00:19,405 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5189ce0ee22f4051b25775f81810d303, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733630414356 2024-12-08T04:00:19,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0f4f2728774414ae8083c2a4f8a291, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733630417750 2024-12-08T04:00:19,405 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting aafe75ab80b54d72a854e96a98260b32, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733630415557 2024-12-08T04:00:19,406 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3bc7d1ba3447828eb566bf29385ae8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733630417750 2024-12-08T04:00:19,417 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#B#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:19,417 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/7340baf73f6b4c10a0af09cc990aa450 is 50, key is test_row_0/B:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:19,420 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:19,425 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412085fa25ce77e1d46eb8be323965566aa8c_00ee06bc7c2f5d203c30b1a99ea00704 store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742210_1386 (size=12595) 2024-12-08T04:00:19,427 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412085fa25ce77e1d46eb8be323965566aa8c_00ee06bc7c2f5d203c30b1a99ea00704, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:19,427 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085fa25ce77e1d46eb8be323965566aa8c_00ee06bc7c2f5d203c30b1a99ea00704 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:19,432 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/7340baf73f6b4c10a0af09cc990aa450 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/7340baf73f6b4c10a0af09cc990aa450 2024-12-08T04:00:19,440 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/B of 00ee06bc7c2f5d203c30b1a99ea00704 into 7340baf73f6b4c10a0af09cc990aa450(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:19,440 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:19,440 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/B, priority=13, startTime=1733630419403; duration=0sec 2024-12-08T04:00:19,440 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:19,440 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:B 2024-12-08T04:00:19,440 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:19,441 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:19,441 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/C is initiating minor compaction (all files) 2024-12-08T04:00:19,441 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/C in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:19,442 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/88ea0822295844bd8d46a52a4c0dd05f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1478453a2c6744ee9ad228a7547406b5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/85084ad64bbb4c9a9443ddd1bbf39fc2] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=35.9 K 2024-12-08T04:00:19,442 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ea0822295844bd8d46a52a4c0dd05f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733630414356 2024-12-08T04:00:19,442 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1478453a2c6744ee9ad228a7547406b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733630415557 2024-12-08T04:00:19,442 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 85084ad64bbb4c9a9443ddd1bbf39fc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733630417750 2024-12-08T04:00:19,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742211_1387 (size=4469) 2024-12-08T04:00:19,449 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:19,449 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/0c88424102f3498eb8a9b292119d60c8 is 50, key is test_row_0/C:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:19,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742212_1388 (size=12595) 2024-12-08T04:00:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:19,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T04:00:19,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:19,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:19,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:19,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:19,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:19,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:19,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083ae0859348424d559d8626aae473073b_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630418935/Put/seqid=0 2024-12-08T04:00:19,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742213_1389 (size=14794) 2024-12-08T04:00:19,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:19,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083ae0859348424d559d8626aae473073b_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083ae0859348424d559d8626aae473073b_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:19,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/4b02df380e174742b49feeb6f8694422, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:19,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/4b02df380e174742b49feeb6f8694422 is 175, key is test_row_0/A:col10/1733630418935/Put/seqid=0 2024-12-08T04:00:19,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742214_1390 (size=39749) 2024-12-08T04:00:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630479615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630479618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630479623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630479624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630479727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630479737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630479737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630479737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,849 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:19,850 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/d6f93528cdf5478e8b9163f46bf887d4 is 175, key is test_row_0/A:col10/1733630417754/Put/seqid=0 2024-12-08T04:00:19,873 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/0c88424102f3498eb8a9b292119d60c8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0c88424102f3498eb8a9b292119d60c8 2024-12-08T04:00:19,878 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/C of 00ee06bc7c2f5d203c30b1a99ea00704 into 0c88424102f3498eb8a9b292119d60c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:19,878 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:19,878 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/C, priority=13, startTime=1733630419403; duration=0sec 2024-12-08T04:00:19,878 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:19,878 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:C 2024-12-08T04:00:19,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742215_1391 (size=31549) 2024-12-08T04:00:19,887 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/d6f93528cdf5478e8b9163f46bf887d4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/d6f93528cdf5478e8b9163f46bf887d4 2024-12-08T04:00:19,892 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/A of 00ee06bc7c2f5d203c30b1a99ea00704 into d6f93528cdf5478e8b9163f46bf887d4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:19,892 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:19,892 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/A, priority=13, startTime=1733630419402; duration=0sec 2024-12-08T04:00:19,892 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:19,892 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:A 2024-12-08T04:00:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630479937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630479948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630479948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630479948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/4b02df380e174742b49feeb6f8694422 2024-12-08T04:00:20,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/1b2ae042c91449d680d95e1c16cfa41a is 50, key is test_row_0/B:col10/1733630418935/Put/seqid=0 2024-12-08T04:00:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742216_1392 (size=12151) 2024-12-08T04:00:20,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/1b2ae042c91449d680d95e1c16cfa41a 2024-12-08T04:00:20,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/9933d9b33940441eb31f345be169da8b is 50, key is test_row_0/C:col10/1733630418935/Put/seqid=0 2024-12-08T04:00:20,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742217_1393 (size=12151) 2024-12-08T04:00:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T04:00:20,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:20,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-08T04:00:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T04:00:20,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:20,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:20,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T04:00:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630480257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630480257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630480269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630480271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-08T04:00:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:20,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:20,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:20,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:20,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-08T04:00:20,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:20,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:20,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:20,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:20,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T04:00:20,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/9933d9b33940441eb31f345be169da8b 2024-12-08T04:00:20,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/4b02df380e174742b49feeb6f8694422 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422 2024-12-08T04:00:20,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422, entries=200, sequenceid=222, filesize=38.8 K 2024-12-08T04:00:20,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/1b2ae042c91449d680d95e1c16cfa41a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1b2ae042c91449d680d95e1c16cfa41a 2024-12-08T04:00:20,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1b2ae042c91449d680d95e1c16cfa41a, entries=150, sequenceid=222, filesize=11.9 K 2024-12-08T04:00:20,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/9933d9b33940441eb31f345be169da8b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/9933d9b33940441eb31f345be169da8b 2024-12-08T04:00:20,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/9933d9b33940441eb31f345be169da8b, entries=150, sequenceid=222, filesize=11.9 K 2024-12-08T04:00:20,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 00ee06bc7c2f5d203c30b1a99ea00704 in 951ms, sequenceid=222, compaction requested=false 2024-12-08T04:00:20,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:20,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-08T04:00:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:20,591 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T04:00:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:20,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:20,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:20,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe6a9cf6459b42459fc16f046052511d_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630419622/Put/seqid=0 2024-12-08T04:00:20,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742218_1394 (size=12304) 2024-12-08T04:00:20,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:20,656 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe6a9cf6459b42459fc16f046052511d_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe6a9cf6459b42459fc16f046052511d_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e8d3babc9d204aef9656d67f031d58ec, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e8d3babc9d204aef9656d67f031d58ec is 175, key is test_row_0/A:col10/1733630419622/Put/seqid=0 2024-12-08T04:00:20,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742219_1395 (size=31105) 2024-12-08T04:00:20,702 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e8d3babc9d204aef9656d67f031d58ec 2024-12-08T04:00:20,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/0cd388e0ae984ea495e4f499ad8df438 is 50, key is test_row_0/B:col10/1733630419622/Put/seqid=0 2024-12-08T04:00:20,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742220_1396 (size=12151) 2024-12-08T04:00:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T04:00:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:20,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630480798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630480800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630480804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630480806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630480917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630480917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630480927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630480928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630481127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630481137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,149 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/0cd388e0ae984ea495e4f499ad8df438 2024-12-08T04:00:21,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630481147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630481147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/56faa590099446e48d91f3eb90db5772 is 50, key is test_row_0/C:col10/1733630419622/Put/seqid=0 2024-12-08T04:00:21,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742221_1397 (size=12151) 2024-12-08T04:00:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T04:00:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630481448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630481458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630481467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630481467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:21,584 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/56faa590099446e48d91f3eb90db5772 2024-12-08T04:00:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e8d3babc9d204aef9656d67f031d58ec as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec 2024-12-08T04:00:21,591 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec, entries=150, sequenceid=244, filesize=30.4 K 2024-12-08T04:00:21,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/0cd388e0ae984ea495e4f499ad8df438 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0cd388e0ae984ea495e4f499ad8df438 2024-12-08T04:00:21,596 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0cd388e0ae984ea495e4f499ad8df438, entries=150, sequenceid=244, filesize=11.9 K 2024-12-08T04:00:21,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/56faa590099446e48d91f3eb90db5772 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/56faa590099446e48d91f3eb90db5772 2024-12-08T04:00:21,603 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/56faa590099446e48d91f3eb90db5772, entries=150, sequenceid=244, filesize=11.9 K 2024-12-08T04:00:21,603 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1012ms, sequenceid=244, compaction requested=true 2024-12-08T04:00:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-08T04:00:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-08T04:00:21,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-08T04:00:21,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4800 sec 2024-12-08T04:00:21,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.4880 sec 2024-12-08T04:00:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:21,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T04:00:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:21,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:21,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dbd7ab8745ad4f55be4668e2178ad75a_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:21,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742222_1398 (size=14844) 2024-12-08T04:00:22,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630482027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630482027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630482028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630482037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630482147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630482147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630482147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630482147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-08T04:00:22,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:22,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-08T04:00:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:22,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:22,280 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:22,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:22,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630482357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630482357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630482367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630482367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:22,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dbd7ab8745ad4f55be4668e2178ad75a_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbd7ab8745ad4f55be4668e2178ad75a_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:22,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/0fadd24fe41c423cbc997ebef2820b32, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:22,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/0fadd24fe41c423cbc997ebef2820b32 is 175, key is test_row_0/A:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:22,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742223_1399 (size=39799) 2024-12-08T04:00:22,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:22,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:22,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630482677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630482677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630482687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630482687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:22,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/0fadd24fe41c423cbc997ebef2820b32 2024-12-08T04:00:22,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/cf494aae55794c3b80653efdb532dc73 is 50, key is test_row_0/B:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:22,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742224_1400 (size=12201) 2024-12-08T04:00:22,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:22,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:23,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630483197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:23,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630483197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630483197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:23,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630483207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:23,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/cf494aae55794c3b80653efdb532dc73 2024-12-08T04:00:23,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1fee955145374f2fb6b169cb53600642 is 50, key is test_row_0/C:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:23,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742225_1401 (size=12201) 2024-12-08T04:00:23,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:23,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:23,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:23,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:23,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:23,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:23,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1fee955145374f2fb6b169cb53600642 2024-12-08T04:00:23,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/0fadd24fe41c423cbc997ebef2820b32 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32 2024-12-08T04:00:23,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32, entries=200, sequenceid=257, filesize=38.9 K 2024-12-08T04:00:23,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/cf494aae55794c3b80653efdb532dc73 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/cf494aae55794c3b80653efdb532dc73 2024-12-08T04:00:23,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/cf494aae55794c3b80653efdb532dc73, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T04:00:23,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/1fee955145374f2fb6b169cb53600642 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1fee955145374f2fb6b169cb53600642 2024-12-08T04:00:23,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1fee955145374f2fb6b169cb53600642, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T04:00:23,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1669ms, sequenceid=257, compaction requested=true 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:23,642 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:23,642 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:23,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:23,643 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:23,643 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/B is initiating minor compaction (all files) 2024-12-08T04:00:23,643 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/B in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,643 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/7340baf73f6b4c10a0af09cc990aa450, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1b2ae042c91449d680d95e1c16cfa41a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0cd388e0ae984ea495e4f499ad8df438, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/cf494aae55794c3b80653efdb532dc73] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=47.9 K 2024-12-08T04:00:23,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142202 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:23,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/A is initiating minor compaction (all files) 2024-12-08T04:00:23,644 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/A in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,644 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/d6f93528cdf5478e8b9163f46bf887d4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=138.9 K 2024-12-08T04:00:23,644 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/d6f93528cdf5478e8b9163f46bf887d4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32] 2024-12-08T04:00:23,644 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 7340baf73f6b4c10a0af09cc990aa450, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733630417750 2024-12-08T04:00:23,646 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f93528cdf5478e8b9163f46bf887d4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733630417750 2024-12-08T04:00:23,647 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2ae042c91449d680d95e1c16cfa41a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733630418935 2024-12-08T04:00:23,648 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd388e0ae984ea495e4f499ad8df438, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733630419616 2024-12-08T04:00:23,648 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b02df380e174742b49feeb6f8694422, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733630418933 2024-12-08T04:00:23,648 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting cf494aae55794c3b80653efdb532dc73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630420798 2024-12-08T04:00:23,648 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d3babc9d204aef9656d67f031d58ec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733630419616 2024-12-08T04:00:23,649 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fadd24fe41c423cbc997ebef2820b32, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630420798 2024-12-08T04:00:23,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-08T04:00:23,658 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:23,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,659 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T04:00:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:23,660 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#B#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:23,660 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/d3fff6abe73c4c10aadf95ea30edd6d6 is 50, key is test_row_0/B:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:23,667 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208b0e4b2b13ae849af8b46fd7566e1157b_00ee06bc7c2f5d203c30b1a99ea00704 store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:23,670 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208b0e4b2b13ae849af8b46fd7566e1157b_00ee06bc7c2f5d203c30b1a99ea00704, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:23,670 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b0e4b2b13ae849af8b46fd7566e1157b_00ee06bc7c2f5d203c30b1a99ea00704 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b3e10ea5c70a4c49a2380af27b444776_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630422013/Put/seqid=0 2024-12-08T04:00:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742226_1402 (size=12781) 2024-12-08T04:00:23,696 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/d3fff6abe73c4c10aadf95ea30edd6d6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/d3fff6abe73c4c10aadf95ea30edd6d6 2024-12-08T04:00:23,701 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/B of 00ee06bc7c2f5d203c30b1a99ea00704 into d3fff6abe73c4c10aadf95ea30edd6d6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:23,701 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:23,701 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/B, priority=12, startTime=1733630423642; duration=0sec 2024-12-08T04:00:23,701 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:23,701 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:B 2024-12-08T04:00:23,702 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:23,703 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:23,703 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/C is initiating minor compaction (all files) 2024-12-08T04:00:23,703 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/C in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:23,703 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0c88424102f3498eb8a9b292119d60c8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/9933d9b33940441eb31f345be169da8b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/56faa590099446e48d91f3eb90db5772, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1fee955145374f2fb6b169cb53600642] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=47.9 K 2024-12-08T04:00:23,704 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c88424102f3498eb8a9b292119d60c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733630417750 2024-12-08T04:00:23,704 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 9933d9b33940441eb31f345be169da8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733630418935 2024-12-08T04:00:23,704 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 56faa590099446e48d91f3eb90db5772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733630419616 2024-12-08T04:00:23,704 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fee955145374f2fb6b169cb53600642, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630420798 2024-12-08T04:00:23,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742227_1403 (size=4469) 2024-12-08T04:00:23,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742228_1404 (size=12454) 2024-12-08T04:00:23,721 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:23,722 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/68a28fc308a14b5eba609fcaf705d0c5 is 50, key is test_row_0/C:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742229_1405 (size=12781) 2024-12-08T04:00:24,105 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#A#compaction#339 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:24,106 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/2f1fd1b5dd1f4ac7a3f2db1688613076 is 175, key is test_row_0/A:col10/1733630420798/Put/seqid=0 2024-12-08T04:00:24,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742230_1406 (size=31735) 2024-12-08T04:00:24,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:24,117 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b3e10ea5c70a4c49a2380af27b444776_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b3e10ea5c70a4c49a2380af27b444776_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/570df96fd0974b188c4ac9ac5c26436e, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/570df96fd0974b188c4ac9ac5c26436e is 175, key is test_row_0/A:col10/1733630422013/Put/seqid=0 2024-12-08T04:00:24,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742231_1407 (size=31255) 2024-12-08T04:00:24,128 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/570df96fd0974b188c4ac9ac5c26436e 2024-12-08T04:00:24,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/abad1a707e904ea08a021a2a0e6b35aa is 50, key is test_row_0/B:col10/1733630422013/Put/seqid=0 2024-12-08T04:00:24,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742232_1408 (size=12301) 2024-12-08T04:00:24,158 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/68a28fc308a14b5eba609fcaf705d0c5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/68a28fc308a14b5eba609fcaf705d0c5 2024-12-08T04:00:24,169 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/C of 00ee06bc7c2f5d203c30b1a99ea00704 into 68a28fc308a14b5eba609fcaf705d0c5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:24,169 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:24,169 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/C, priority=12, startTime=1733630423642; duration=0sec 2024-12-08T04:00:24,169 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:24,169 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:C 2024-12-08T04:00:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:24,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. as already flushing 2024-12-08T04:00:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630484248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630484253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630484262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630484263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630484367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630484378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630484378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630484378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56744 deadline: 1733630484427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,438 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in 00ee06bc7c2f5d203c30b1a99ea00704 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,516 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/2f1fd1b5dd1f4ac7a3f2db1688613076 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f1fd1b5dd1f4ac7a3f2db1688613076 2024-12-08T04:00:24,521 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/A of 00ee06bc7c2f5d203c30b1a99ea00704 into 2f1fd1b5dd1f4ac7a3f2db1688613076(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:24,521 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:24,521 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/A, priority=12, startTime=1733630423642; duration=0sec 2024-12-08T04:00:24,521 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:24,521 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:A 2024-12-08T04:00:24,545 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/abad1a707e904ea08a021a2a0e6b35aa 2024-12-08T04:00:24,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/b95644efcc4c4278a175ce9431602884 is 50, key is test_row_0/C:col10/1733630422013/Put/seqid=0 2024-12-08T04:00:24,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742233_1409 (size=12301) 2024-12-08T04:00:24,556 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/b95644efcc4c4278a175ce9431602884 2024-12-08T04:00:24,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/570df96fd0974b188c4ac9ac5c26436e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e 2024-12-08T04:00:24,563 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e, entries=150, sequenceid=276, filesize=30.5 K 2024-12-08T04:00:24,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/abad1a707e904ea08a021a2a0e6b35aa as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/abad1a707e904ea08a021a2a0e6b35aa 2024-12-08T04:00:24,569 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/abad1a707e904ea08a021a2a0e6b35aa, entries=150, sequenceid=276, filesize=12.0 K 2024-12-08T04:00:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/b95644efcc4c4278a175ce9431602884 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/b95644efcc4c4278a175ce9431602884 2024-12-08T04:00:24,575 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/b95644efcc4c4278a175ce9431602884, entries=150, sequenceid=276, filesize=12.0 K 2024-12-08T04:00:24,576 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 00ee06bc7c2f5d203c30b1a99ea00704 in 917ms, sequenceid=276, compaction requested=false 2024-12-08T04:00:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-08T04:00:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-08T04:00:24,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-08T04:00:24,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2970 sec 2024-12-08T04:00:24,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.3030 sec 2024-12-08T04:00:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:24,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T04:00:24,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:24,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:24,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:24,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:24,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:24,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:24,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120820d31547f9264820b5cf6c37db8290ae_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:24,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742234_1410 (size=14994) 2024-12-08T04:00:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630484620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630484621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630484627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630484627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630484737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630484737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630484737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630484737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630484947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630484947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630484947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630484947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:25,015 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120820d31547f9264820b5cf6c37db8290ae_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120820d31547f9264820b5cf6c37db8290ae_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:25,015 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/00f820d4f4264dc8b93174050f3c3971, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:25,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/00f820d4f4264dc8b93174050f3c3971 is 175, key is test_row_0/A:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742235_1411 (size=39949) 2024-12-08T04:00:25,188 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-12-08T04:00:25,188 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:25,188 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,188 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:25,189 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-12-08T04:00:25,189 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:25,189 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,189 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:25,197 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-12-08T04:00:25,198 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:25,198 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,198 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:25,199 INFO [Thread-1532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1532. 2024-12-08T04:00:25,199 DEBUG [Thread-1532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:25,199 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,199 DEBUG [Thread-1532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:25,199 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-12-08T04:00:25,199 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:25,199 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,199 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:25,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:25,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630485258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630485258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630485267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630485267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/00f820d4f4264dc8b93174050f3c3971 2024-12-08T04:00:25,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/8cb5454066864f5aa5f0e2a8b7fc1583 is 50, key is test_row_0/B:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742236_1412 (size=12301) 2024-12-08T04:00:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56792 deadline: 1733630485767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56816 deadline: 1733630485767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56778 deadline: 1733630485778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56800 deadline: 1733630485778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:00:25,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/8cb5454066864f5aa5f0e2a8b7fc1583 2024-12-08T04:00:25,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/69c5ccdfc8374f348c2374dd7f463419 is 50, key is test_row_0/C:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:25,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742237_1413 (size=12301) 2024-12-08T04:00:26,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/69c5ccdfc8374f348c2374dd7f463419 2024-12-08T04:00:26,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/00f820d4f4264dc8b93174050f3c3971 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971 2024-12-08T04:00:26,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971, entries=200, sequenceid=294, filesize=39.0 K 2024-12-08T04:00:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/8cb5454066864f5aa5f0e2a8b7fc1583 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8cb5454066864f5aa5f0e2a8b7fc1583 2024-12-08T04:00:26,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8cb5454066864f5aa5f0e2a8b7fc1583, entries=150, sequenceid=294, filesize=12.0 K 2024-12-08T04:00:26,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/69c5ccdfc8374f348c2374dd7f463419 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/69c5ccdfc8374f348c2374dd7f463419 2024-12-08T04:00:26,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/69c5ccdfc8374f348c2374dd7f463419, entries=150, sequenceid=294, filesize=12.0 K 2024-12-08T04:00:26,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1686ms, sequenceid=294, compaction requested=true 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:26,275 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00ee06bc7c2f5d203c30b1a99ea00704:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:26,275 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/B is initiating minor compaction (all files) 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/A is initiating minor compaction (all files) 2024-12-08T04:00:26,276 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/B in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:26,276 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/A in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:26,276 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/d3fff6abe73c4c10aadf95ea30edd6d6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/abad1a707e904ea08a021a2a0e6b35aa, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8cb5454066864f5aa5f0e2a8b7fc1583] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=36.5 K 2024-12-08T04:00:26,276 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f1fd1b5dd1f4ac7a3f2db1688613076, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=100.5 K 2024-12-08T04:00:26,276 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f1fd1b5dd1f4ac7a3f2db1688613076, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971] 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d3fff6abe73c4c10aadf95ea30edd6d6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630420798 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f1fd1b5dd1f4ac7a3f2db1688613076, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630420798 2024-12-08T04:00:26,276 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting abad1a707e904ea08a021a2a0e6b35aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733630422013 2024-12-08T04:00:26,277 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 570df96fd0974b188c4ac9ac5c26436e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733630422013 2024-12-08T04:00:26,277 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb5454066864f5aa5f0e2a8b7fc1583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733630424243 2024-12-08T04:00:26,277 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f820d4f4264dc8b93174050f3c3971, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733630424243 2024-12-08T04:00:26,283 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:26,284 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:26,284 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/10ec84c085604a0b88c8a380daa5a58c is 50, key is test_row_0/B:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:26,285 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208fc66346a5435451eb9ed27446f5c47c2_00ee06bc7c2f5d203c30b1a99ea00704 store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:26,289 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208fc66346a5435451eb9ed27446f5c47c2_00ee06bc7c2f5d203c30b1a99ea00704, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:26,289 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fc66346a5435451eb9ed27446f5c47c2_00ee06bc7c2f5d203c30b1a99ea00704 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:26,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742238_1414 (size=12983) 2024-12-08T04:00:26,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742239_1415 (size=4469) 2024-12-08T04:00:26,298 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/10ec84c085604a0b88c8a380daa5a58c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/10ec84c085604a0b88c8a380daa5a58c 2024-12-08T04:00:26,299 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#A#compaction#348 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:26,299 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e95948cea33b4da99dda5aff8a421996 is 175, key is test_row_0/A:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:26,304 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/B of 00ee06bc7c2f5d203c30b1a99ea00704 into 10ec84c085604a0b88c8a380daa5a58c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:26,304 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:26,304 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/B, priority=13, startTime=1733630426275; duration=0sec 2024-12-08T04:00:26,304 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:26,304 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:B 2024-12-08T04:00:26,304 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:26,304 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:26,305 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 00ee06bc7c2f5d203c30b1a99ea00704/C is initiating minor compaction (all files) 2024-12-08T04:00:26,305 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00ee06bc7c2f5d203c30b1a99ea00704/C in TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:26,305 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/68a28fc308a14b5eba609fcaf705d0c5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/b95644efcc4c4278a175ce9431602884, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/69c5ccdfc8374f348c2374dd7f463419] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp, totalSize=36.5 K 2024-12-08T04:00:26,305 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a28fc308a14b5eba609fcaf705d0c5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630420798 2024-12-08T04:00:26,305 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b95644efcc4c4278a175ce9431602884, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733630422013 2024-12-08T04:00:26,306 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c5ccdfc8374f348c2374dd7f463419, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733630424243 2024-12-08T04:00:26,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742240_1416 (size=31937) 2024-12-08T04:00:26,318 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00ee06bc7c2f5d203c30b1a99ea00704#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:26,319 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/16e5f73202524762b4f4d7c2ba45a870 is 50, key is test_row_0/C:col10/1733630424588/Put/seqid=0 2024-12-08T04:00:26,322 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/e95948cea33b4da99dda5aff8a421996 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e95948cea33b4da99dda5aff8a421996 2024-12-08T04:00:26,328 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/A of 00ee06bc7c2f5d203c30b1a99ea00704 into e95948cea33b4da99dda5aff8a421996(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:26,328 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:26,328 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/A, priority=13, startTime=1733630426275; duration=0sec 2024-12-08T04:00:26,328 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:26,328 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:A 2024-12-08T04:00:26,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742241_1417 (size=12983) 2024-12-08T04:00:26,343 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/16e5f73202524762b4f4d7c2ba45a870 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/16e5f73202524762b4f4d7c2ba45a870 2024-12-08T04:00:26,348 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00ee06bc7c2f5d203c30b1a99ea00704/C of 00ee06bc7c2f5d203c30b1a99ea00704 into 16e5f73202524762b4f4d7c2ba45a870(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:26,348 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:26,348 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., storeName=00ee06bc7c2f5d203c30b1a99ea00704/C, priority=13, startTime=1733630426275; duration=0sec 2024-12-08T04:00:26,348 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:26,348 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00ee06bc7c2f5d203c30b1a99ea00704:C 2024-12-08T04:00:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T04:00:26,438 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:26,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T04:00:26,778 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-12-08T04:00:26,778 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-12-08T04:00:26,778 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:26,778 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:26,778 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,778 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,779 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,779 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:26,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:26,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:26,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:26,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:26,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:26,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:26,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:26,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080ef4ff082f0b4c81b2308986bdca2a62_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630426777/Put/seqid=0 2024-12-08T04:00:26,789 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-12-08T04:00:26,789 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-12-08T04:00:26,789 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:26,789 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:26,789 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,789 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,789 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,789 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:26,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:26,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:26,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742242_1418 (size=12454) 2024-12-08T04:00:26,889 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T04:00:27,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:27,195 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080ef4ff082f0b4c81b2308986bdca2a62_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080ef4ff082f0b4c81b2308986bdca2a62_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:27,196 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/7a955a9f3c634175af506148666434bd, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:27,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/7a955a9f3c634175af506148666434bd is 175, key is test_row_0/A:col10/1733630426777/Put/seqid=0 2024-12-08T04:00:27,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742243_1419 (size=31255) 2024-12-08T04:00:27,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/7a955a9f3c634175af506148666434bd 2024-12-08T04:00:27,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/f96c707406464976980f07f23d66d4df is 50, key is test_row_0/B:col10/1733630426777/Put/seqid=0 2024-12-08T04:00:27,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742244_1420 (size=12301) 2024-12-08T04:00:28,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/f96c707406464976980f07f23d66d4df 2024-12-08T04:00:28,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/a097e55fbf514cd3b112934694414a33 is 50, key is test_row_0/C:col10/1733630426777/Put/seqid=0 2024-12-08T04:00:28,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742245_1421 (size=12301) 2024-12-08T04:00:28,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/a097e55fbf514cd3b112934694414a33 2024-12-08T04:00:28,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/7a955a9f3c634175af506148666434bd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/7a955a9f3c634175af506148666434bd 2024-12-08T04:00:28,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/7a955a9f3c634175af506148666434bd, entries=150, sequenceid=316, filesize=30.5 K 2024-12-08T04:00:28,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/f96c707406464976980f07f23d66d4df as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/f96c707406464976980f07f23d66d4df 2024-12-08T04:00:28,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/f96c707406464976980f07f23d66d4df, entries=150, sequenceid=316, filesize=12.0 K 2024-12-08T04:00:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/a097e55fbf514cd3b112934694414a33 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/a097e55fbf514cd3b112934694414a33 2024-12-08T04:00:28,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/a097e55fbf514cd3b112934694414a33, entries=150, sequenceid=316, filesize=12.0 K 2024-12-08T04:00:28,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1659ms, sequenceid=316, compaction requested=false 2024-12-08T04:00:28,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00ee06bc7c2f5d203c30b1a99ea00704: 2024-12-08T04:00:31,391 WARN [regionserver/32df4b4bb439:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 4, running: 1 2024-12-08T04:00:31,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/776baeec5c8e4a5287c34ef1c04093db, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5189ce0ee22f4051b25775f81810d303, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/d6f93528cdf5478e8b9163f46bf887d4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f1fd1b5dd1f4ac7a3f2db1688613076, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971] to archive 2024-12-08T04:00:31,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:00:31,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/8b4f63c41a3f47cc95734ce34f5af6d8 2024-12-08T04:00:31,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/09db63100afe4b7f9e5ac28dad206b10 2024-12-08T04:00:31,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c724471424d84127923be25bf5361f49 2024-12-08T04:00:31,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/df093072fefe49349247aa61306b5a9c 2024-12-08T04:00:31,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e72d240f1c2c42fe83981902ebbc3cc3 2024-12-08T04:00:31,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f29c57301104ede8317f9d5ef5f7d04 2024-12-08T04:00:31,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/c957194732f64b1aa75e20268746e845 2024-12-08T04:00:31,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/776baeec5c8e4a5287c34ef1c04093db to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/776baeec5c8e4a5287c34ef1c04093db 2024-12-08T04:00:31,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/da96cd37d4bf431288f8209329070193 2024-12-08T04:00:31,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/72e2fa7614b54bf4bba73ad135ec622b 2024-12-08T04:00:31,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5189ce0ee22f4051b25775f81810d303 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5189ce0ee22f4051b25775f81810d303 2024-12-08T04:00:31,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/aafe75ab80b54d72a854e96a98260b32 2024-12-08T04:00:31,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/d6f93528cdf5478e8b9163f46bf887d4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/d6f93528cdf5478e8b9163f46bf887d4 2024-12-08T04:00:31,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/5e3bc7d1ba3447828eb566bf29385ae8 2024-12-08T04:00:31,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/4b02df380e174742b49feeb6f8694422 2024-12-08T04:00:31,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e8d3babc9d204aef9656d67f031d58ec 2024-12-08T04:00:31,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/0fadd24fe41c423cbc997ebef2820b32 2024-12-08T04:00:31,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f1fd1b5dd1f4ac7a3f2db1688613076 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/2f1fd1b5dd1f4ac7a3f2db1688613076 2024-12-08T04:00:31,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/570df96fd0974b188c4ac9ac5c26436e 2024-12-08T04:00:31,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/00f820d4f4264dc8b93174050f3c3971 2024-12-08T04:00:31,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/e690811b0dbd41bfa579aef16a434fae, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6dbe0ec86f57406c9b6026ed471dd309, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/689047e763944f8dbb925451578ec5bf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ecd92e54892e44e2aea07ce93e778384, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/347aa37ec59c45a884e8dd52999e24d8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0e091c493297415ca8a8f3cb06be2538, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6190ccfe29bc4575b2472eef4bf46719, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ba91961a88eb4870a5104649bfc6431e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/dc5531c4942b405c9187dcd4c2f7e537, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/83303d17659a4e5db0d8f9eae30dd5e2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1159f92fdab544b991e9b0cee0f32913, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8136126956c74eacad237937c94a8432, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/7340baf73f6b4c10a0af09cc990aa450, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ea0f4f2728774414ae8083c2a4f8a291, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1b2ae042c91449d680d95e1c16cfa41a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0cd388e0ae984ea495e4f499ad8df438, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/d3fff6abe73c4c10aadf95ea30edd6d6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/cf494aae55794c3b80653efdb532dc73, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/abad1a707e904ea08a021a2a0e6b35aa, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8cb5454066864f5aa5f0e2a8b7fc1583] to archive 2024-12-08T04:00:31,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:00:31,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/e690811b0dbd41bfa579aef16a434fae to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/e690811b0dbd41bfa579aef16a434fae 2024-12-08T04:00:31,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6dbe0ec86f57406c9b6026ed471dd309 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6dbe0ec86f57406c9b6026ed471dd309 2024-12-08T04:00:31,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/689047e763944f8dbb925451578ec5bf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/689047e763944f8dbb925451578ec5bf 2024-12-08T04:00:31,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ecd92e54892e44e2aea07ce93e778384 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ecd92e54892e44e2aea07ce93e778384 2024-12-08T04:00:31,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/347aa37ec59c45a884e8dd52999e24d8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/347aa37ec59c45a884e8dd52999e24d8 2024-12-08T04:00:31,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0e091c493297415ca8a8f3cb06be2538 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0e091c493297415ca8a8f3cb06be2538 2024-12-08T04:00:31,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6190ccfe29bc4575b2472eef4bf46719 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/6190ccfe29bc4575b2472eef4bf46719 2024-12-08T04:00:31,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ba91961a88eb4870a5104649bfc6431e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ba91961a88eb4870a5104649bfc6431e 2024-12-08T04:00:31,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/dc5531c4942b405c9187dcd4c2f7e537 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/dc5531c4942b405c9187dcd4c2f7e537 2024-12-08T04:00:31,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/83303d17659a4e5db0d8f9eae30dd5e2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/83303d17659a4e5db0d8f9eae30dd5e2 2024-12-08T04:00:31,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1159f92fdab544b991e9b0cee0f32913 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1159f92fdab544b991e9b0cee0f32913 2024-12-08T04:00:31,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8136126956c74eacad237937c94a8432 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8136126956c74eacad237937c94a8432 2024-12-08T04:00:31,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/7340baf73f6b4c10a0af09cc990aa450 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/7340baf73f6b4c10a0af09cc990aa450 2024-12-08T04:00:31,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ea0f4f2728774414ae8083c2a4f8a291 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/ea0f4f2728774414ae8083c2a4f8a291 2024-12-08T04:00:31,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1b2ae042c91449d680d95e1c16cfa41a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/1b2ae042c91449d680d95e1c16cfa41a 2024-12-08T04:00:31,448 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0cd388e0ae984ea495e4f499ad8df438 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/0cd388e0ae984ea495e4f499ad8df438 2024-12-08T04:00:31,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/d3fff6abe73c4c10aadf95ea30edd6d6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/d3fff6abe73c4c10aadf95ea30edd6d6 2024-12-08T04:00:31,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/cf494aae55794c3b80653efdb532dc73 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/cf494aae55794c3b80653efdb532dc73 2024-12-08T04:00:31,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/abad1a707e904ea08a021a2a0e6b35aa to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/abad1a707e904ea08a021a2a0e6b35aa 2024-12-08T04:00:31,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8cb5454066864f5aa5f0e2a8b7fc1583 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/8cb5454066864f5aa5f0e2a8b7fc1583 2024-12-08T04:00:31,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/f4f215492e0c4c04a453ab3c95fc5dc2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/48b4cbd5ede446f9b2196d1496d31b9d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/71082c17817b421d95ce664c0f76ecf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/605f1d2b93884517a2135e49a32e3e37, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/687d676070644ebda8034da6c693264a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/feddd737704942eab275e42918c39441, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/5e3c94c77d1443dfb2aa0f4200fffc82, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0410697a4dc94b2e966b5d50a9d2b43d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1bd6d00664ab4d239bff95544835410a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/88ea0822295844bd8d46a52a4c0dd05f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/651d210c0f4048c791f7a798bd352ded, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1478453a2c6744ee9ad228a7547406b5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0c88424102f3498eb8a9b292119d60c8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/85084ad64bbb4c9a9443ddd1bbf39fc2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/9933d9b33940441eb31f345be169da8b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/56faa590099446e48d91f3eb90db5772, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/68a28fc308a14b5eba609fcaf705d0c5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1fee955145374f2fb6b169cb53600642, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/b95644efcc4c4278a175ce9431602884, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/69c5ccdfc8374f348c2374dd7f463419] to archive 2024-12-08T04:00:31,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:00:31,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/f4f215492e0c4c04a453ab3c95fc5dc2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/f4f215492e0c4c04a453ab3c95fc5dc2 2024-12-08T04:00:31,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/48b4cbd5ede446f9b2196d1496d31b9d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/48b4cbd5ede446f9b2196d1496d31b9d 2024-12-08T04:00:31,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/71082c17817b421d95ce664c0f76ecf7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/71082c17817b421d95ce664c0f76ecf7 2024-12-08T04:00:31,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/605f1d2b93884517a2135e49a32e3e37 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/605f1d2b93884517a2135e49a32e3e37 2024-12-08T04:00:31,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/687d676070644ebda8034da6c693264a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/687d676070644ebda8034da6c693264a 2024-12-08T04:00:31,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/feddd737704942eab275e42918c39441 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/feddd737704942eab275e42918c39441 2024-12-08T04:00:31,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/5e3c94c77d1443dfb2aa0f4200fffc82 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/5e3c94c77d1443dfb2aa0f4200fffc82 2024-12-08T04:00:31,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0410697a4dc94b2e966b5d50a9d2b43d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0410697a4dc94b2e966b5d50a9d2b43d 2024-12-08T04:00:31,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1bd6d00664ab4d239bff95544835410a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1bd6d00664ab4d239bff95544835410a 2024-12-08T04:00:31,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/88ea0822295844bd8d46a52a4c0dd05f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/88ea0822295844bd8d46a52a4c0dd05f 2024-12-08T04:00:31,466 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/651d210c0f4048c791f7a798bd352ded to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/651d210c0f4048c791f7a798bd352ded 2024-12-08T04:00:31,466 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1478453a2c6744ee9ad228a7547406b5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1478453a2c6744ee9ad228a7547406b5 2024-12-08T04:00:31,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0c88424102f3498eb8a9b292119d60c8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/0c88424102f3498eb8a9b292119d60c8 2024-12-08T04:00:31,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/85084ad64bbb4c9a9443ddd1bbf39fc2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/85084ad64bbb4c9a9443ddd1bbf39fc2 2024-12-08T04:00:31,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/9933d9b33940441eb31f345be169da8b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/9933d9b33940441eb31f345be169da8b 2024-12-08T04:00:31,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/56faa590099446e48d91f3eb90db5772 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/56faa590099446e48d91f3eb90db5772 2024-12-08T04:00:31,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/68a28fc308a14b5eba609fcaf705d0c5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/68a28fc308a14b5eba609fcaf705d0c5 2024-12-08T04:00:31,470 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1fee955145374f2fb6b169cb53600642 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/1fee955145374f2fb6b169cb53600642 2024-12-08T04:00:31,471 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/b95644efcc4c4278a175ce9431602884 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/b95644efcc4c4278a175ce9431602884 2024-12-08T04:00:31,471 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/69c5ccdfc8374f348c2374dd7f463419 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/69c5ccdfc8374f348c2374dd7f463419 2024-12-08T04:00:34,478 INFO [Thread-1521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1521. 2024-12-08T04:00:34,478 DEBUG [Thread-1521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:34,478 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:34,478 DEBUG [Thread-1521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:34,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1401 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4193 rows 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1400 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4196 rows 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1410 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4223 rows 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1408 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4200 rows 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1403 2024-12-08T04:00:34,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4209 rows 2024-12-08T04:00:34,479 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:00:34,479 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:00:34,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:34,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:34,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:34,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T04:00:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T04:00:34,483 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630434483"}]},"ts":"1733630434483"} 2024-12-08T04:00:34,484 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T04:00:34,485 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T04:00:34,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T04:00:34,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, UNASSIGN}] 2024-12-08T04:00:34,487 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, UNASSIGN 2024-12-08T04:00:34,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:34,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, UNASSIGN because future has completed 2024-12-08T04:00:34,489 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T04:00:34,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:00:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T04:00:34,642 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:34,642 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T04:00:34,642 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 00ee06bc7c2f5d203c30b1a99ea00704, disabling compactions & flushes 2024-12-08T04:00:34,642 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:34,642 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:34,642 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. after waiting 0 ms 2024-12-08T04:00:34,642 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:34,643 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 00ee06bc7c2f5d203c30b1a99ea00704 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T04:00:34,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=A 2024-12-08T04:00:34,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:34,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=B 2024-12-08T04:00:34,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:34,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00ee06bc7c2f5d203c30b1a99ea00704, store=C 2024-12-08T04:00:34,643 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:34,648 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dbcaadca4b264331ab7064ce81ef158f_00ee06bc7c2f5d203c30b1a99ea00704 is 50, key is test_row_0/A:col10/1733630434477/Put/seqid=0 2024-12-08T04:00:34,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742246_1422 (size=9914) 2024-12-08T04:00:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T04:00:35,052 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:35,058 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208dbcaadca4b264331ab7064ce81ef158f_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbcaadca4b264331ab7064ce81ef158f_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:35,058 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/1fefcfdb1cd14594819350f95b7b70c4, store: [table=TestAcidGuarantees family=A region=00ee06bc7c2f5d203c30b1a99ea00704] 2024-12-08T04:00:35,059 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/1fefcfdb1cd14594819350f95b7b70c4 is 175, key is test_row_0/A:col10/1733630434477/Put/seqid=0 2024-12-08T04:00:35,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742247_1423 (size=22561) 2024-12-08T04:00:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T04:00:35,463 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/1fefcfdb1cd14594819350f95b7b70c4 2024-12-08T04:00:35,476 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/09bc09188cc2487a84c1b0ed68d64e25 is 50, key is test_row_0/B:col10/1733630434477/Put/seqid=0 2024-12-08T04:00:35,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742248_1424 (size=9857) 2024-12-08T04:00:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T04:00:35,881 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/09bc09188cc2487a84c1b0ed68d64e25 2024-12-08T04:00:35,893 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/dd372591156647bfa04394d4746f7437 is 50, key is test_row_0/C:col10/1733630434477/Put/seqid=0 2024-12-08T04:00:35,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742249_1425 (size=9857) 2024-12-08T04:00:36,297 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/dd372591156647bfa04394d4746f7437 2024-12-08T04:00:36,308 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/A/1fefcfdb1cd14594819350f95b7b70c4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/1fefcfdb1cd14594819350f95b7b70c4 2024-12-08T04:00:36,314 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/1fefcfdb1cd14594819350f95b7b70c4, entries=100, sequenceid=322, filesize=22.0 K 2024-12-08T04:00:36,314 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/B/09bc09188cc2487a84c1b0ed68d64e25 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/09bc09188cc2487a84c1b0ed68d64e25 2024-12-08T04:00:36,318 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/09bc09188cc2487a84c1b0ed68d64e25, entries=100, sequenceid=322, filesize=9.6 K 2024-12-08T04:00:36,318 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/.tmp/C/dd372591156647bfa04394d4746f7437 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/dd372591156647bfa04394d4746f7437 2024-12-08T04:00:36,321 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/dd372591156647bfa04394d4746f7437, entries=100, sequenceid=322, filesize=9.6 K 2024-12-08T04:00:36,322 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1680ms, sequenceid=322, compaction requested=true 2024-12-08T04:00:36,326 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-12-08T04:00:36,326 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. 2024-12-08T04:00:36,326 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 00ee06bc7c2f5d203c30b1a99ea00704: Waiting for close lock at 1733630434642Running coprocessor pre-close hooks at 1733630434642Disabling compacts and flushes for region at 1733630434642Disabling writes for close at 1733630434642Obtaining lock to block concurrent updates at 1733630434643 (+1 ms)Preparing flush snapshotting stores in 00ee06bc7c2f5d203c30b1a99ea00704 at 1733630434643Finished memstore snapshotting TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733630434643Flushing stores of TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704. at 1733630434643Flushing 00ee06bc7c2f5d203c30b1a99ea00704/A: creating writer at 1733630434643Flushing mob file 00ee06bc7c2f5d203c30b1a99ea00704/A: appending metadata at 1733630434648 (+5 ms)Flushing mob file 00ee06bc7c2f5d203c30b1a99ea00704/A: closing flushed file at 1733630434648Flushing 00ee06bc7c2f5d203c30b1a99ea00704/A: appending metadata at 1733630435058 (+410 ms)Flushing 00ee06bc7c2f5d203c30b1a99ea00704/A: closing flushed file at 1733630435058Flushing 00ee06bc7c2f5d203c30b1a99ea00704/B: creating writer at 1733630435472 (+414 ms)Flushing 00ee06bc7c2f5d203c30b1a99ea00704/B: appending metadata at 1733630435475 (+3 ms)Flushing 00ee06bc7c2f5d203c30b1a99ea00704/B: closing flushed file at 1733630435475Flushing 00ee06bc7c2f5d203c30b1a99ea00704/C: creating writer at 1733630435888 (+413 ms)Flushing 00ee06bc7c2f5d203c30b1a99ea00704/C: appending metadata at 1733630435892 (+4 ms)Flushing 00ee06bc7c2f5d203c30b1a99ea00704/C: closing flushed file at 1733630435892Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1f6369: reopening flushed file at 1733630436306 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@172195ab: reopening flushed file at 1733630436314 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f50afb8: reopening flushed file at 1733630436318 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00ee06bc7c2f5d203c30b1a99ea00704 in 1680ms, sequenceid=322, compaction requested=true at 1733630436322 (+4 ms)Writing region close event to WAL at 1733630436323 (+1 ms)Running coprocessor post-close hooks at 1733630436326 (+3 ms)Closed at 1733630436326 2024-12-08T04:00:36,328 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,329 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=00ee06bc7c2f5d203c30b1a99ea00704, regionState=CLOSED 2024-12-08T04:00:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:00:36,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-08T04:00:36,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 00ee06bc7c2f5d203c30b1a99ea00704, server=32df4b4bb439,39919,1733630308479 in 1.8410 sec 2024-12-08T04:00:36,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-08T04:00:36,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00ee06bc7c2f5d203c30b1a99ea00704, UNASSIGN in 1.8460 sec 2024-12-08T04:00:36,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-08T04:00:36,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8490 sec 2024-12-08T04:00:36,336 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630436336"}]},"ts":"1733630436336"} 2024-12-08T04:00:36,337 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T04:00:36,337 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T04:00:36,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8570 sec 2024-12-08T04:00:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-08T04:00:36,627 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:36,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T04:00:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T04:00:36,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,632 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/recovered.edits] 2024-12-08T04:00:36,637 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/1fefcfdb1cd14594819350f95b7b70c4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/1fefcfdb1cd14594819350f95b7b70c4 2024-12-08T04:00:36,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/7a955a9f3c634175af506148666434bd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/7a955a9f3c634175af506148666434bd 2024-12-08T04:00:36,640 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e95948cea33b4da99dda5aff8a421996 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/A/e95948cea33b4da99dda5aff8a421996 2024-12-08T04:00:36,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/09bc09188cc2487a84c1b0ed68d64e25 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/09bc09188cc2487a84c1b0ed68d64e25 2024-12-08T04:00:36,643 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/10ec84c085604a0b88c8a380daa5a58c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/10ec84c085604a0b88c8a380daa5a58c 2024-12-08T04:00:36,644 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/f96c707406464976980f07f23d66d4df to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/B/f96c707406464976980f07f23d66d4df 2024-12-08T04:00:36,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/16e5f73202524762b4f4d7c2ba45a870 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/16e5f73202524762b4f4d7c2ba45a870 2024-12-08T04:00:36,648 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/a097e55fbf514cd3b112934694414a33 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/a097e55fbf514cd3b112934694414a33 2024-12-08T04:00:36,649 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/dd372591156647bfa04394d4746f7437 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/C/dd372591156647bfa04394d4746f7437 2024-12-08T04:00:36,651 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/recovered.edits/325.seqid to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704/recovered.edits/325.seqid 2024-12-08T04:00:36,651 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,651 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T04:00:36,652 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T04:00:36,652 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T04:00:36,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208055cab27a4734c43a09f55a86e518259_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208055cab27a4734c43a09f55a86e518259_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,656 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080ef4ff082f0b4c81b2308986bdca2a62_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080ef4ff082f0b4c81b2308986bdca2a62_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,657 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081c971a7cd4cc4849a8ed53bffdcf76e9_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081c971a7cd4cc4849a8ed53bffdcf76e9_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,658 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120820d31547f9264820b5cf6c37db8290ae_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120820d31547f9264820b5cf6c37db8290ae_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,659 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083ae0859348424d559d8626aae473073b_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083ae0859348424d559d8626aae473073b_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,661 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083d131084e5c14b4cad2ae8ebc0d0ac90_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412083d131084e5c14b4cad2ae8ebc0d0ac90_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,661 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853ba8783faad49489cc83bf7e7ead74b_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120853ba8783faad49489cc83bf7e7ead74b_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,662 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120877e079390f3141e09264e5de1f4f1974_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120877e079390f3141e09264e5de1f4f1974_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,664 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b3e10ea5c70a4c49a2380af27b444776_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b3e10ea5c70a4c49a2380af27b444776_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,665 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d17c454ba43540f4966e9bc944f61b31_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d17c454ba43540f4966e9bc944f61b31_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,666 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d835daba22564410970c56a5173094f3_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d835daba22564410970c56a5173094f3_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,667 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbcaadca4b264331ab7064ce81ef158f_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbcaadca4b264331ab7064ce81ef158f_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbd7ab8745ad4f55be4668e2178ad75a_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208dbd7ab8745ad4f55be4668e2178ad75a_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,669 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e6d6dcd3398a48ac90524115b30bc0dc_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e6d6dcd3398a48ac90524115b30bc0dc_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,669 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ee88318945dc4ffa8be8d725097f9446_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ee88318945dc4ffa8be8d725097f9446_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,670 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f899ea6c55984f84b05d4024bd17cce2_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f899ea6c55984f84b05d4024bd17cce2_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,672 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f8d862040e8f48f9bb3c9f71d856b204_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208f8d862040e8f48f9bb3c9f71d856b204_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,673 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe6a9cf6459b42459fc16f046052511d_00ee06bc7c2f5d203c30b1a99ea00704 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe6a9cf6459b42459fc16f046052511d_00ee06bc7c2f5d203c30b1a99ea00704 2024-12-08T04:00:36,673 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T04:00:36,675 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,677 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T04:00:36,679 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T04:00:36,680 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,680 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T04:00:36,680 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733630436680"}]},"ts":"9223372036854775807"} 2024-12-08T04:00:36,682 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T04:00:36,682 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00ee06bc7c2f5d203c30b1a99ea00704, NAME => 'TestAcidGuarantees,,1733630402056.00ee06bc7c2f5d203c30b1a99ea00704.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T04:00:36,682 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T04:00:36,683 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733630436682"}]},"ts":"9223372036854775807"} 2024-12-08T04:00:36,684 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T04:00:36,685 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-08T04:00:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T04:00:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T04:00:36,738 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:36,744 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 248), OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 453), ProcessCount=11 (was 11), AvailableMemoryMB=16977 (was 16268) - AvailableMemoryMB LEAK? - 2024-12-08T04:00:36,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=16977 2024-12-08T04:00:36,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T04:00:36,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:00:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T04:00:36,754 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T04:00:36,754 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:36,754 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-12-08T04:00:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-08T04:00:36,755 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T04:00:36,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742250_1426 (size=1061) 2024-12-08T04:00:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-08T04:00:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-08T04:00:37,162 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T04:00:37,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742251_1427 (size=53) 2024-12-08T04:00:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-08T04:00:37,570 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:00:37,570 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5f0fa7839174f51942b3285bec09c3ac, disabling compactions & flushes 2024-12-08T04:00:37,570 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,570 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,570 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. after waiting 0 ms 2024-12-08T04:00:37,570 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,571 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f0fa7839174f51942b3285bec09c3ac: Waiting for close lock at 1733630437570Disabling compacts and flushes for region at 1733630437570Disabling writes for close at 1733630437570Writing region close event to WAL at 1733630437570Closed at 1733630437570 2024-12-08T04:00:37,572 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T04:00:37,573 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733630437572"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630437572"}]},"ts":"1733630437572"} 2024-12-08T04:00:37,575 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T04:00:37,576 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T04:00:37,576 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630437576"}]},"ts":"1733630437576"} 2024-12-08T04:00:37,578 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T04:00:37,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, ASSIGN}] 2024-12-08T04:00:37,580 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, ASSIGN 2024-12-08T04:00:37,581 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, ASSIGN; state=OFFLINE, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=false 2024-12-08T04:00:37,732 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=5f0fa7839174f51942b3285bec09c3ac, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:37,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, ASSIGN because future has completed 2024-12-08T04:00:37,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:00:37,891 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,891 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:00:37,892 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,892 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:00:37,892 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,892 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,894 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,896 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:37,897 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f0fa7839174f51942b3285bec09c3ac columnFamilyName A 2024-12-08T04:00:37,897 DEBUG [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-08T04:00:37,898 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.HStore(327): Store=5f0fa7839174f51942b3285bec09c3ac/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:37,898 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,900 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:37,900 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f0fa7839174f51942b3285bec09c3ac columnFamilyName B 2024-12-08T04:00:37,900 DEBUG [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:37,901 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.HStore(327): Store=5f0fa7839174f51942b3285bec09c3ac/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:37,901 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,902 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:00:37,903 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f0fa7839174f51942b3285bec09c3ac columnFamilyName C 2024-12-08T04:00:37,903 DEBUG [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:00:37,903 INFO [StoreOpener-5f0fa7839174f51942b3285bec09c3ac-1 {}] regionserver.HStore(327): Store=5f0fa7839174f51942b3285bec09c3ac/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:00:37,903 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,904 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,904 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,905 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,906 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,906 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,907 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T04:00:37,908 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,911 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:00:37,911 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 5f0fa7839174f51942b3285bec09c3ac; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60636017, jitterRate=-0.09645293653011322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T04:00:37,912 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:37,912 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 5f0fa7839174f51942b3285bec09c3ac: Running coprocessor pre-open hook at 1733630437893Writing region info on filesystem at 1733630437893Initializing all the Stores at 1733630437894 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630437894Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630437894Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630437894Cleaning up temporary data from old regions at 1733630437906 (+12 ms)Running coprocessor post-open hooks at 1733630437912 (+6 ms)Region opened successfully at 1733630437912 2024-12-08T04:00:37,914 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., pid=122, masterSystemTime=1733630437888 2024-12-08T04:00:37,916 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,916 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:37,917 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=5f0fa7839174f51942b3285bec09c3ac, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:37,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:00:37,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-08T04:00:37,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 in 184 msec 2024-12-08T04:00:37,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-12-08T04:00:37,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, ASSIGN in 342 msec 2024-12-08T04:00:37,923 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T04:00:37,923 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630437923"}]},"ts":"1733630437923"} 2024-12-08T04:00:37,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T04:00:37,925 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T04:00:37,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-08T04:00:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-08T04:00:38,908 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:38,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,912 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3136302e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,916 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4442d924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,918 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,923 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T04:00:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T04:00:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43ce90aa 2024-12-08T04:00:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T04:00:38,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T04:00:38,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1243265c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,928 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71899dd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631806cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3443de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,932 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c97f4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,934 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@186c2dbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,936 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a93d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4502da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,940 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4576773d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7655f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,943 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a912e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb3e13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b08f110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406e7507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9efcc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb5c0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c61afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0ba93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:00:38,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:00:38,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:00:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:00:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:00:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337cd38a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:00:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:00:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:38,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:00:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd8b67e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:00:38,959 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,959 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,959 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,959 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,959 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,959 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,959 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:38,960 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,960 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,960 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-08T04:00:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,962 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:00:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:00:38,963 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:00:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,965 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:38,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:00:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2] 2024-12-08T04:00:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:38,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:00:38,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:38,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:38,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:38,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:38,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:38,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:38,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d96d0ed90e754438ba297b71b9921811 is 50, key is test_row_0/A:col10/1733630438971/Put/seqid=0 2024-12-08T04:00:39,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630498996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630498997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630438994, max timestamp=1733630439003Num uniques 859; 2024-12-08T04:00:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630498997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630438994, max timestamp=1733630439003Num uniques 859; 2024-12-08T04:00:39,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630438994, max timestamp=1733630439003Num uniques 859; 2024-12-08T04:00:39,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:39,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:39,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742252_1428 (size=11997) 2024-12-08T04:00:39,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d96d0ed90e754438ba297b71b9921811 2024-12-08T04:00:39,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/1092d2ac355844f883de40df4fd43536 is 50, key is test_row_0/B:col10/1733630438971/Put/seqid=0 2024-12-08T04:00:39,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742253_1429 (size=9657) 2024-12-08T04:00:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:39,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630499062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630499066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630499107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630499107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:39,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:39,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630499117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630499177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630499177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:39,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:39,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:39,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630499317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630499317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630499337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630499387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630499387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/1092d2ac355844f883de40df4fd43536 2024-12-08T04:00:39,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/32d7d12d7779421db195880826d0290f is 50, key is test_row_0/C:col10/1733630438971/Put/seqid=0 2024-12-08T04:00:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742254_1430 (size=9657) 2024-12-08T04:00:39,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:39,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:39,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630499627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630499628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630499647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630499697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630499697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:39,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:39,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:39,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/32d7d12d7779421db195880826d0290f 2024-12-08T04:00:39,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d96d0ed90e754438ba297b71b9921811 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d96d0ed90e754438ba297b71b9921811 2024-12-08T04:00:39,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d96d0ed90e754438ba297b71b9921811, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T04:00:39,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/1092d2ac355844f883de40df4fd43536 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/1092d2ac355844f883de40df4fd43536 2024-12-08T04:00:39,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/1092d2ac355844f883de40df4fd43536, entries=100, sequenceid=12, filesize=9.4 K 2024-12-08T04:00:39,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:39,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/32d7d12d7779421db195880826d0290f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/32d7d12d7779421db195880826d0290f 2024-12-08T04:00:39,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:39,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:39,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:39,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/32d7d12d7779421db195880826d0290f, entries=100, sequenceid=12, filesize=9.4 K 2024-12-08T04:00:39,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5f0fa7839174f51942b3285bec09c3ac in 913ms, sequenceid=12, compaction requested=false 2024-12-08T04:00:39,886 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T04:00:39,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:40,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T04:00:40,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:40,035 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-08T04:00:40,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:40,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:40,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/4d565923545b4596bb7ced9298f51a1f is 50, key is test_row_0/A:col10/1733630439052/Put/seqid=0 2024-12-08T04:00:40,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742255_1431 (size=12001) 2024-12-08T04:00:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630500157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630500165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630500165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630500207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630500207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630500277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630500277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,442 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/4d565923545b4596bb7ced9298f51a1f 2024-12-08T04:00:40,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/06e524b01b6145c7bb03a453dc1744b0 is 50, key is test_row_0/B:col10/1733630439052/Put/seqid=0 2024-12-08T04:00:40,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742256_1432 (size=12001) 2024-12-08T04:00:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630500487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630500487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630500797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630500797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:40,851 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/06e524b01b6145c7bb03a453dc1744b0 2024-12-08T04:00:40,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/36b134ff7b434146adb1a80b4a93a51d is 50, key is test_row_0/C:col10/1733630439052/Put/seqid=0 2024-12-08T04:00:40,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742257_1433 (size=12001) 2024-12-08T04:00:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:41,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630501177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,186 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630501217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630501217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,225 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,225 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,261 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/36b134ff7b434146adb1a80b4a93a51d 2024-12-08T04:00:41,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/4d565923545b4596bb7ced9298f51a1f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4d565923545b4596bb7ced9298f51a1f 2024-12-08T04:00:41,268 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4d565923545b4596bb7ced9298f51a1f, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T04:00:41,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/06e524b01b6145c7bb03a453dc1744b0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/06e524b01b6145c7bb03a453dc1744b0 2024-12-08T04:00:41,271 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/06e524b01b6145c7bb03a453dc1744b0, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T04:00:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/36b134ff7b434146adb1a80b4a93a51d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/36b134ff7b434146adb1a80b4a93a51d 2024-12-08T04:00:41,274 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/36b134ff7b434146adb1a80b4a93a51d, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T04:00:41,275 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 5f0fa7839174f51942b3285bec09c3ac in 1240ms, sequenceid=47, compaction requested=false 2024-12-08T04:00:41,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:41,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:41,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-08T04:00:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-08T04:00:41,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-08T04:00:41,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3110 sec 2024-12-08T04:00:41,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3200 sec 2024-12-08T04:00:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:41,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T04:00:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:41,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/49396407358849aa9a93335fb790fad7 is 50, key is test_row_0/A:col10/1733630441321/Put/seqid=0 2024-12-08T04:00:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742258_1434 (size=14341) 2024-12-08T04:00:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630501372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630501375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630501487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630501497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630501697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630501707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:41,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/49396407358849aa9a93335fb790fad7 2024-12-08T04:00:41,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9fe0230840204ab680cc9c68bdb80272 is 50, key is test_row_0/B:col10/1733630441321/Put/seqid=0 2024-12-08T04:00:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742259_1435 (size=12001) 2024-12-08T04:00:41,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T04:00:42,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630502007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:42,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630502017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:42,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9fe0230840204ab680cc9c68bdb80272 2024-12-08T04:00:42,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/7010273f8c5a43ecaf628bec8e9e820d is 50, key is test_row_0/C:col10/1733630441321/Put/seqid=0 2024-12-08T04:00:42,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742260_1436 (size=12001) 2024-12-08T04:00:42,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630502517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:42,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630502527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:42,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/7010273f8c5a43ecaf628bec8e9e820d 2024-12-08T04:00:42,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/49396407358849aa9a93335fb790fad7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/49396407358849aa9a93335fb790fad7 2024-12-08T04:00:42,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/49396407358849aa9a93335fb790fad7, entries=200, sequenceid=61, filesize=14.0 K 2024-12-08T04:00:42,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9fe0230840204ab680cc9c68bdb80272 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9fe0230840204ab680cc9c68bdb80272 2024-12-08T04:00:42,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9fe0230840204ab680cc9c68bdb80272, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T04:00:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/7010273f8c5a43ecaf628bec8e9e820d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/7010273f8c5a43ecaf628bec8e9e820d 2024-12-08T04:00:42,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/7010273f8c5a43ecaf628bec8e9e820d, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T04:00:42,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5f0fa7839174f51942b3285bec09c3ac in 1254ms, sequenceid=61, compaction requested=true 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:42,576 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:42,576 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:42,577 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:42,577 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:42,577 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d96d0ed90e754438ba297b71b9921811, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4d565923545b4596bb7ced9298f51a1f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/49396407358849aa9a93335fb790fad7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=37.4 K 2024-12-08T04:00:42,577 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/1092d2ac355844f883de40df4fd43536, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/06e524b01b6145c7bb03a453dc1744b0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9fe0230840204ab680cc9c68bdb80272] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=32.9 K 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1092d2ac355844f883de40df4fd43536, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733630438971 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting d96d0ed90e754438ba297b71b9921811, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733630438968 2024-12-08T04:00:42,577 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d565923545b4596bb7ced9298f51a1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733630439052 2024-12-08T04:00:42,578 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e524b01b6145c7bb03a453dc1744b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733630439052 2024-12-08T04:00:42,578 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49396407358849aa9a93335fb790fad7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733630440163 2024-12-08T04:00:42,578 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe0230840204ab680cc9c68bdb80272, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733630440163 2024-12-08T04:00:42,584 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:42,584 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:42,584 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9bcb2c1e3f674d4f90346fe33a82d654 is 50, key is test_row_0/B:col10/1733630441321/Put/seqid=0 2024-12-08T04:00:42,585 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/51ecf6f259f44aefb6b3e04bc976aaeb is 50, key is test_row_0/A:col10/1733630441321/Put/seqid=0 2024-12-08T04:00:42,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742261_1437 (size=12104) 2024-12-08T04:00:42,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742262_1438 (size=12104) 2024-12-08T04:00:42,609 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/51ecf6f259f44aefb6b3e04bc976aaeb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/51ecf6f259f44aefb6b3e04bc976aaeb 2024-12-08T04:00:42,613 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into 51ecf6f259f44aefb6b3e04bc976aaeb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:42,613 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:42,613 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=13, startTime=1733630442576; duration=0sec 2024-12-08T04:00:42,613 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:42,613 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:42,614 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:42,614 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:42,614 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:42,614 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:42,614 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/32d7d12d7779421db195880826d0290f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/36b134ff7b434146adb1a80b4a93a51d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/7010273f8c5a43ecaf628bec8e9e820d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=32.9 K 2024-12-08T04:00:42,615 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d7d12d7779421db195880826d0290f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733630438971 2024-12-08T04:00:42,615 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b134ff7b434146adb1a80b4a93a51d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733630439052 2024-12-08T04:00:42,615 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7010273f8c5a43ecaf628bec8e9e820d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733630440163 2024-12-08T04:00:42,621 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:42,622 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/1d43b7d000e5404c861c2211bcb770f2 is 50, key is test_row_0/C:col10/1733630441321/Put/seqid=0 2024-12-08T04:00:42,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742263_1439 (size=12104) 2024-12-08T04:00:42,996 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9bcb2c1e3f674d4f90346fe33a82d654 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9bcb2c1e3f674d4f90346fe33a82d654 2024-12-08T04:00:42,999 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into 9bcb2c1e3f674d4f90346fe33a82d654(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:42,999 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:42,999 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=13, startTime=1733630442576; duration=0sec 2024-12-08T04:00:42,999 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:42,999 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:43,031 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/1d43b7d000e5404c861c2211bcb770f2 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1d43b7d000e5404c861c2211bcb770f2 2024-12-08T04:00:43,036 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into 1d43b7d000e5404c861c2211bcb770f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:43,036 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:43,036 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=13, startTime=1733630442576; duration=0sec 2024-12-08T04:00:43,036 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:43,036 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T04:00:43,127 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:43,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-08T04:00:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:43,129 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:43,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:43,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:43,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T04:00:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:43,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:43,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/031673738e834b6eb83d0abf198717da is 50, key is test_row_0/A:col10/1733630443199/Put/seqid=0 2024-12-08T04:00:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:43,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742264_1440 (size=14341) 2024-12-08T04:00:43,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630503257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630503261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630503261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630503367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630503367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630503377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630503537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,541 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630503547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,549 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630503576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630503587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630503597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/031673738e834b6eb83d0abf198717da 2024-12-08T04:00:43,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/20adbb22b29c4fb29b719ea46c477473 is 50, key is test_row_0/B:col10/1733630443199/Put/seqid=0 2024-12-08T04:00:43,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742265_1441 (size=12001) 2024-12-08T04:00:43,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:43,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:43,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:43,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:43,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630503888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630503897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630503907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:44,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/20adbb22b29c4fb29b719ea46c477473 2024-12-08T04:00:44,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/1e880e9a0d51416d9c53fa7bbb8c9ede is 50, key is test_row_0/C:col10/1733630443199/Put/seqid=0 2024-12-08T04:00:44,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742266_1442 (size=12001) 2024-12-08T04:00:44,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:44,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:44,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630504407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630504417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:44,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630504417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:44,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/1e880e9a0d51416d9c53fa7bbb8c9ede 2024-12-08T04:00:44,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/031673738e834b6eb83d0abf198717da as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/031673738e834b6eb83d0abf198717da 2024-12-08T04:00:44,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/031673738e834b6eb83d0abf198717da, entries=200, sequenceid=83, filesize=14.0 K 2024-12-08T04:00:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/20adbb22b29c4fb29b719ea46c477473 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/20adbb22b29c4fb29b719ea46c477473 2024-12-08T04:00:44,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/20adbb22b29c4fb29b719ea46c477473, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T04:00:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/1e880e9a0d51416d9c53fa7bbb8c9ede as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1e880e9a0d51416d9c53fa7bbb8c9ede 2024-12-08T04:00:44,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1e880e9a0d51416d9c53fa7bbb8c9ede, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T04:00:44,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5f0fa7839174f51942b3285bec09c3ac in 1283ms, sequenceid=83, compaction requested=false 2024-12-08T04:00:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:44,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T04:00:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:44,508 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T04:00:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/34231a1328c9408f95067e05a8993ccf is 50, key is test_row_0/A:col10/1733630443260/Put/seqid=0 2024-12-08T04:00:44,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742267_1443 (size=12001) 2024-12-08T04:00:44,916 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/34231a1328c9408f95067e05a8993ccf 2024-12-08T04:00:44,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/fcce68535cb64ab8aeb2cdcb563a0632 is 50, key is test_row_0/B:col10/1733630443260/Put/seqid=0 2024-12-08T04:00:44,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742268_1444 (size=12001) 2024-12-08T04:00:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:45,334 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/fcce68535cb64ab8aeb2cdcb563a0632 2024-12-08T04:00:45,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/9770a099367e4a5bb58f83a3920d452e is 50, key is test_row_0/C:col10/1733630443260/Put/seqid=0 2024-12-08T04:00:45,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742269_1445 (size=12001) 2024-12-08T04:00:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:45,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:45,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630505459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630505465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630505466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630505567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630505567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,571 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,572 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630505577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630505577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630505577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,742 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/9770a099367e4a5bb58f83a3920d452e 2024-12-08T04:00:45,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/34231a1328c9408f95067e05a8993ccf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/34231a1328c9408f95067e05a8993ccf 2024-12-08T04:00:45,748 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/34231a1328c9408f95067e05a8993ccf, entries=150, sequenceid=96, filesize=11.7 K 2024-12-08T04:00:45,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/fcce68535cb64ab8aeb2cdcb563a0632 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/fcce68535cb64ab8aeb2cdcb563a0632 2024-12-08T04:00:45,751 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/fcce68535cb64ab8aeb2cdcb563a0632, entries=150, sequenceid=96, filesize=11.7 K 2024-12-08T04:00:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/9770a099367e4a5bb58f83a3920d452e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/9770a099367e4a5bb58f83a3920d452e 2024-12-08T04:00:45,754 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/9770a099367e4a5bb58f83a3920d452e, entries=150, sequenceid=96, filesize=11.7 K 2024-12-08T04:00:45,755 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5f0fa7839174f51942b3285bec09c3ac in 1247ms, sequenceid=96, compaction requested=true 2024-12-08T04:00:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-08T04:00:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-08T04:00:45,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-08T04:00:45,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6260 sec 2024-12-08T04:00:45,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6300 sec 2024-12-08T04:00:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:45,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T04:00:45,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:45,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:45,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/ec2eff0e705d462c9529b20baba5f242 is 50, key is test_row_0/A:col10/1733630445457/Put/seqid=0 2024-12-08T04:00:45,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742270_1446 (size=14341) 2024-12-08T04:00:45,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630505803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630505804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630505810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630505917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630505918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630505918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630506137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630506138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630506138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/ec2eff0e705d462c9529b20baba5f242 2024-12-08T04:00:46,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9017d7de912648c4ab398370b826a0dc is 50, key is test_row_0/B:col10/1733630445457/Put/seqid=0 2024-12-08T04:00:46,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742271_1447 (size=12001) 2024-12-08T04:00:46,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9017d7de912648c4ab398370b826a0dc 2024-12-08T04:00:46,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/f62c483ccfe6478f8a9cb98cd0e8c592 is 50, key is test_row_0/C:col10/1733630445457/Put/seqid=0 2024-12-08T04:00:46,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742272_1448 (size=12001) 2024-12-08T04:00:46,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/f62c483ccfe6478f8a9cb98cd0e8c592 2024-12-08T04:00:46,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/ec2eff0e705d462c9529b20baba5f242 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/ec2eff0e705d462c9529b20baba5f242 2024-12-08T04:00:46,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/ec2eff0e705d462c9529b20baba5f242, entries=200, sequenceid=116, filesize=14.0 K 2024-12-08T04:00:46,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/9017d7de912648c4ab398370b826a0dc as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9017d7de912648c4ab398370b826a0dc 2024-12-08T04:00:46,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9017d7de912648c4ab398370b826a0dc, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T04:00:46,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/f62c483ccfe6478f8a9cb98cd0e8c592 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f62c483ccfe6478f8a9cb98cd0e8c592 2024-12-08T04:00:46,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f62c483ccfe6478f8a9cb98cd0e8c592, entries=150, sequenceid=116, filesize=11.7 K 2024-12-08T04:00:46,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5f0fa7839174f51942b3285bec09c3ac in 528ms, sequenceid=116, compaction requested=true 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:46,321 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:46,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T04:00:46,321 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:46,322 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:46,322 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:46,322 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:46,322 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/51ecf6f259f44aefb6b3e04bc976aaeb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/031673738e834b6eb83d0abf198717da, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/34231a1328c9408f95067e05a8993ccf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/ec2eff0e705d462c9529b20baba5f242] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=51.5 K 2024-12-08T04:00:46,325 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ecf6f259f44aefb6b3e04bc976aaeb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733630440163 2024-12-08T04:00:46,325 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 031673738e834b6eb83d0abf198717da, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733630441341 2024-12-08T04:00:46,326 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 34231a1328c9408f95067e05a8993ccf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733630443255 2024-12-08T04:00:46,326 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2eff0e705d462c9529b20baba5f242, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733630445451 2024-12-08T04:00:46,327 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:46,327 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:46,327 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:46,327 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9bcb2c1e3f674d4f90346fe33a82d654, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/20adbb22b29c4fb29b719ea46c477473, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/fcce68535cb64ab8aeb2cdcb563a0632, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9017d7de912648c4ab398370b826a0dc] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=47.0 K 2024-12-08T04:00:46,328 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bcb2c1e3f674d4f90346fe33a82d654, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733630440163 2024-12-08T04:00:46,328 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20adbb22b29c4fb29b719ea46c477473, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733630441341 2024-12-08T04:00:46,329 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcce68535cb64ab8aeb2cdcb563a0632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733630443255 2024-12-08T04:00:46,331 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9017d7de912648c4ab398370b826a0dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733630445457 2024-12-08T04:00:46,346 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:46,347 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/942f4f981daf4665804e81ab5e0acd79 is 50, key is test_row_0/A:col10/1733630445457/Put/seqid=0 2024-12-08T04:00:46,347 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:46,347 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/3a77f58faade4c208cb08038a3b28015 is 50, key is test_row_0/B:col10/1733630445457/Put/seqid=0 2024-12-08T04:00:46,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742273_1449 (size=12241) 2024-12-08T04:00:46,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742274_1450 (size=12241) 2024-12-08T04:00:46,393 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/942f4f981daf4665804e81ab5e0acd79 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/942f4f981daf4665804e81ab5e0acd79 2024-12-08T04:00:46,402 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into 942f4f981daf4665804e81ab5e0acd79(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:46,402 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:46,402 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=12, startTime=1733630446321; duration=0sec 2024-12-08T04:00:46,402 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:46,402 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:46,402 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:46,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:46,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:46,404 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:46,404 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1d43b7d000e5404c861c2211bcb770f2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1e880e9a0d51416d9c53fa7bbb8c9ede, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/9770a099367e4a5bb58f83a3920d452e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f62c483ccfe6478f8a9cb98cd0e8c592] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=47.0 K 2024-12-08T04:00:46,404 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d43b7d000e5404c861c2211bcb770f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733630440163 2024-12-08T04:00:46,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e880e9a0d51416d9c53fa7bbb8c9ede, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733630441341 2024-12-08T04:00:46,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 9770a099367e4a5bb58f83a3920d452e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733630443255 2024-12-08T04:00:46,405 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f62c483ccfe6478f8a9cb98cd0e8c592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733630445457 2024-12-08T04:00:46,417 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:46,418 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/027804ae51a243258132c0faba795e5e is 50, key is test_row_0/C:col10/1733630445457/Put/seqid=0 2024-12-08T04:00:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:46,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T04:00:46,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:46,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:46,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:46,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:46,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:46,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:46,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742275_1451 (size=12241) 2024-12-08T04:00:46,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/f4c92b25a27949418e000119a14ba7cd is 50, key is test_row_0/A:col10/1733630445795/Put/seqid=0 2024-12-08T04:00:46,471 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/027804ae51a243258132c0faba795e5e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/027804ae51a243258132c0faba795e5e 2024-12-08T04:00:46,483 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into 027804ae51a243258132c0faba795e5e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:46,483 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:46,484 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=12, startTime=1733630446321; duration=0sec 2024-12-08T04:00:46,484 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:46,484 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:46,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742276_1452 (size=14391) 2024-12-08T04:00:46,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630506517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630506530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630506532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630506639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630506648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630506648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,772 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/3a77f58faade4c208cb08038a3b28015 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/3a77f58faade4c208cb08038a3b28015 2024-12-08T04:00:46,782 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into 3a77f58faade4c208cb08038a3b28015(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:46,782 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:46,782 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=12, startTime=1733630446321; duration=0sec 2024-12-08T04:00:46,783 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:46,783 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:46,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630506858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630506858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630506867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:46,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/f4c92b25a27949418e000119a14ba7cd 2024-12-08T04:00:46,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/f2ea115d5c04494194eb6956db4c9fbc is 50, key is test_row_0/B:col10/1733630445795/Put/seqid=0 2024-12-08T04:00:46,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742277_1453 (size=12051) 2024-12-08T04:00:46,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/f2ea115d5c04494194eb6956db4c9fbc 2024-12-08T04:00:46,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/4004fa79243741ba9742505fa100d710 is 50, key is test_row_0/C:col10/1733630445795/Put/seqid=0 2024-12-08T04:00:46,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742278_1454 (size=12051) 2024-12-08T04:00:46,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/4004fa79243741ba9742505fa100d710 2024-12-08T04:00:46,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/f4c92b25a27949418e000119a14ba7cd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f4c92b25a27949418e000119a14ba7cd 2024-12-08T04:00:46,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f4c92b25a27949418e000119a14ba7cd, entries=200, sequenceid=130, filesize=14.1 K 2024-12-08T04:00:46,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/f2ea115d5c04494194eb6956db4c9fbc as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/f2ea115d5c04494194eb6956db4c9fbc 2024-12-08T04:00:46,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/f2ea115d5c04494194eb6956db4c9fbc, entries=150, sequenceid=130, filesize=11.8 K 2024-12-08T04:00:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/4004fa79243741ba9742505fa100d710 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/4004fa79243741ba9742505fa100d710 2024-12-08T04:00:47,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/4004fa79243741ba9742505fa100d710, entries=150, sequenceid=130, filesize=11.8 K 2024-12-08T04:00:47,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5f0fa7839174f51942b3285bec09c3ac in 550ms, sequenceid=130, compaction requested=false 2024-12-08T04:00:47,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:47,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-08T04:00:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733630446511, max timestamp=1733630447174Num uniques 859; 2024-12-08T04:00:47,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:47,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630446511, max timestamp=1733630447174Num uniques 859; 2024-12-08T04:00:47,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:00:47,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:47,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630446511, max timestamp=1733630447174Num uniques 859; 2024-12-08T04:00:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:47,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:00:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:47,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/5b9ccf6af0254371b084195bc8b1efd1 is 50, key is test_row_0/A:col10/1733630446528/Put/seqid=0 2024-12-08T04:00:47,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742279_1455 (size=14541) 2024-12-08T04:00:47,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/5b9ccf6af0254371b084195bc8b1efd1 2024-12-08T04:00:47,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/bf368a9d316f4f94857f848c4128d938 is 50, key is test_row_0/B:col10/1733630446528/Put/seqid=0 2024-12-08T04:00:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630447236, max timestamp=1733630447258Num uniques 859; 2024-12-08T04:00:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630447236, max timestamp=1733630447258Num uniques 859; 2024-12-08T04:00:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630447236, max timestamp=1733630447258Num uniques 859; 2024-12-08T04:00:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742280_1456 (size=12151) 2024-12-08T04:00:47,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/bf368a9d316f4f94857f848c4128d938 2024-12-08T04:00:47,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/f256cc1c9c564c1b909b45b364db40e1 is 50, key is test_row_0/C:col10/1733630446528/Put/seqid=0 2024-12-08T04:00:47,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742281_1457 (size=12151) 2024-12-08T04:00:47,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/f256cc1c9c564c1b909b45b364db40e1 2024-12-08T04:00:47,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/5b9ccf6af0254371b084195bc8b1efd1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/5b9ccf6af0254371b084195bc8b1efd1 2024-12-08T04:00:47,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/5b9ccf6af0254371b084195bc8b1efd1, entries=200, sequenceid=153, filesize=14.2 K 2024-12-08T04:00:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/bf368a9d316f4f94857f848c4128d938 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bf368a9d316f4f94857f848c4128d938 2024-12-08T04:00:47,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bf368a9d316f4f94857f848c4128d938, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T04:00:47,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/f256cc1c9c564c1b909b45b364db40e1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f256cc1c9c564c1b909b45b364db40e1 2024-12-08T04:00:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T04:00:47,308 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:47,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-08T04:00:47,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f256cc1c9c564c1b909b45b364db40e1, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T04:00:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T04:00:47,311 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:47,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~29.07 KB/29770, heapSize ~71.27 KB/72976, currentSize=100.63 KB/103050 for 5f0fa7839174f51942b3285bec09c3ac in 135ms, sequenceid=153, compaction requested=true 2024-12-08T04:00:47,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:47,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:47,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:47,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:47,311 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:47,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:47,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:47,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T04:00:47,312 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:47,312 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:47,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:47,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=107.34 KB heapSize=300.61 KB 2024-12-08T04:00:47,313 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:47,313 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:47,313 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:47,313 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/3a77f58faade4c208cb08038a3b28015, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/f2ea115d5c04494194eb6956db4c9fbc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bf368a9d316f4f94857f848c4128d938] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=35.6 K 2024-12-08T04:00:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:47,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:47,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:47,314 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:47,314 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:47,314 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,314 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/942f4f981daf4665804e81ab5e0acd79, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f4c92b25a27949418e000119a14ba7cd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/5b9ccf6af0254371b084195bc8b1efd1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=40.2 K 2024-12-08T04:00:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:47,316 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 942f4f981daf4665804e81ab5e0acd79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733630445457 2024-12-08T04:00:47,316 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a77f58faade4c208cb08038a3b28015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733630445457 2024-12-08T04:00:47,316 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c92b25a27949418e000119a14ba7cd, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733630445795 2024-12-08T04:00:47,317 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ea115d5c04494194eb6956db4c9fbc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733630445795 2024-12-08T04:00:47,317 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9ccf6af0254371b084195bc8b1efd1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733630446511 2024-12-08T04:00:47,317 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting bf368a9d316f4f94857f848c4128d938, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733630446528 2024-12-08T04:00:47,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/48a6f589bc29469aa668e670442b767f is 50, key is test_row_0/A:col10/1733630447308/Put/seqid=0 2024-12-08T04:00:47,343 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:47,344 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/84662480ba8245f5b51972f558a20da0 is 50, key is test_row_0/B:col10/1733630446528/Put/seqid=0 2024-12-08T04:00:47,353 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:47,354 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/db8a142b5a714cb3a0ee19cf79b00730 is 50, key is test_row_0/A:col10/1733630446528/Put/seqid=0 2024-12-08T04:00:47,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742282_1458 (size=19321) 2024-12-08T04:00:47,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742283_1459 (size=12493) 2024-12-08T04:00:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630507371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742284_1460 (size=12493) 2024-12-08T04:00:47,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630507389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,405 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/db8a142b5a714cb3a0ee19cf79b00730 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/db8a142b5a714cb3a0ee19cf79b00730 2024-12-08T04:00:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630507390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,416 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into db8a142b5a714cb3a0ee19cf79b00730(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:47,416 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:47,416 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=13, startTime=1733630447311; duration=0sec 2024-12-08T04:00:47,416 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:47,416 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:47,416 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:47,417 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:47,417 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:47,417 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,417 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/027804ae51a243258132c0faba795e5e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/4004fa79243741ba9742505fa100d710, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f256cc1c9c564c1b909b45b364db40e1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=35.6 K 2024-12-08T04:00:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T04:00:47,419 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027804ae51a243258132c0faba795e5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733630445457 2024-12-08T04:00:47,420 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4004fa79243741ba9742505fa100d710, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733630445795 2024-12-08T04:00:47,420 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f256cc1c9c564c1b909b45b364db40e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733630446528 2024-12-08T04:00:47,436 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:47,437 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/b2ca9d663e0b4c929f6efa73965b5a14 is 50, key is test_row_0/C:col10/1733630446528/Put/seqid=0 2024-12-08T04:00:47,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:47,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742285_1461 (size=12493) 2024-12-08T04:00:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,482 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/b2ca9d663e0b4c929f6efa73965b5a14 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b2ca9d663e0b4c929f6efa73965b5a14 2024-12-08T04:00:47,488 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into b2ca9d663e0b4c929f6efa73965b5a14(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:47,488 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:47,488 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=13, startTime=1733630447312; duration=0sec 2024-12-08T04:00:47,488 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:47,488 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630507497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630507508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630507517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T04:00:47,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630507708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630507727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630507738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:47,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/48a6f589bc29469aa668e670442b767f 2024-12-08T04:00:47,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:47,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:47,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/dc3626b79e764929921b963a498aafaf is 50, key is test_row_0/B:col10/1733630447308/Put/seqid=0 2024-12-08T04:00:47,794 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/84662480ba8245f5b51972f558a20da0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/84662480ba8245f5b51972f558a20da0 2024-12-08T04:00:47,802 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into 84662480ba8245f5b51972f558a20da0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:47,802 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:47,802 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=13, startTime=1733630447311; duration=0sec 2024-12-08T04:00:47,802 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:47,802 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742286_1462 (size=12151) 2024-12-08T04:00:47,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:47,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T04:00:48,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630508017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630508047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630508047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:48,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:48,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:48,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/dc3626b79e764929921b963a498aafaf 2024-12-08T04:00:48,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/636d7002119e4bc69dd9ddb314aa69eb is 50, key is test_row_0/C:col10/1733630447308/Put/seqid=0 2024-12-08T04:00:48,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742287_1463 (size=12151) 2024-12-08T04:00:48,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/636d7002119e4bc69dd9ddb314aa69eb 2024-12-08T04:00:48,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/48a6f589bc29469aa668e670442b767f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/48a6f589bc29469aa668e670442b767f 2024-12-08T04:00:48,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/48a6f589bc29469aa668e670442b767f, entries=300, sequenceid=185, filesize=18.9 K 2024-12-08T04:00:48,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/dc3626b79e764929921b963a498aafaf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/dc3626b79e764929921b963a498aafaf 2024-12-08T04:00:48,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/dc3626b79e764929921b963a498aafaf, entries=150, sequenceid=185, filesize=11.9 K 2024-12-08T04:00:48,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/636d7002119e4bc69dd9ddb314aa69eb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/636d7002119e4bc69dd9ddb314aa69eb 2024-12-08T04:00:48,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:48,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:48,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/636d7002119e4bc69dd9ddb314aa69eb, entries=150, sequenceid=185, filesize=11.9 K 2024-12-08T04:00:48,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300.56 KB/307776, currentSize=80.51 KB/82440 for 5f0fa7839174f51942b3285bec09c3ac in 931ms, sequenceid=185, compaction requested=false 2024-12-08T04:00:48,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:48,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:48,401 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/c02ef0b28b494bbdad9534fcf75c268e is 50, key is test_row_0/A:col10/1733630447387/Put/seqid=0 2024-12-08T04:00:48,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742288_1464 (size=12151) 2024-12-08T04:00:48,420 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/c02ef0b28b494bbdad9534fcf75c268e 2024-12-08T04:00:48,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/949f51dae2b74d5dbdd1e34bb8f635e5 is 50, key is test_row_0/B:col10/1733630447387/Put/seqid=0 2024-12-08T04:00:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T04:00:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742289_1465 (size=12151) 2024-12-08T04:00:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:48,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630508583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630508585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630508591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630508698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630508698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630508707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,849 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/949f51dae2b74d5dbdd1e34bb8f635e5 2024-12-08T04:00:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/12abdf787d5548efa1b9c580a08b2a54 is 50, key is test_row_0/C:col10/1733630447387/Put/seqid=0 2024-12-08T04:00:48,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742290_1466 (size=12151) 2024-12-08T04:00:48,880 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/12abdf787d5548efa1b9c580a08b2a54 2024-12-08T04:00:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/c02ef0b28b494bbdad9534fcf75c268e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/c02ef0b28b494bbdad9534fcf75c268e 2024-12-08T04:00:48,893 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/c02ef0b28b494bbdad9534fcf75c268e, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T04:00:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/949f51dae2b74d5dbdd1e34bb8f635e5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/949f51dae2b74d5dbdd1e34bb8f635e5 2024-12-08T04:00:48,897 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/949f51dae2b74d5dbdd1e34bb8f635e5, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T04:00:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/12abdf787d5548efa1b9c580a08b2a54 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/12abdf787d5548efa1b9c580a08b2a54 2024-12-08T04:00:48,903 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/12abdf787d5548efa1b9c580a08b2a54, entries=150, sequenceid=203, filesize=11.9 K 2024-12-08T04:00:48,903 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5f0fa7839174f51942b3285bec09c3ac in 502ms, sequenceid=203, compaction requested=true 2024-12-08T04:00:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-08T04:00:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-08T04:00:48,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-08T04:00:48,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5930 sec 2024-12-08T04:00:48,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5990 sec 2024-12-08T04:00:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:48,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T04:00:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:48,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/e4adedeaf3e04dffa111af3b12014952 is 50, key is test_row_0/A:col10/1733630448580/Put/seqid=0 2024-12-08T04:00:48,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742291_1467 (size=14541) 2024-12-08T04:00:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630508956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630508960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630508964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:48,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630509067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630509077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630509077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630509287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630509296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630509296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/e4adedeaf3e04dffa111af3b12014952 2024-12-08T04:00:49,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/4762c7143f6e410783564c29352df8bf is 50, key is test_row_0/B:col10/1733630448580/Put/seqid=0 2024-12-08T04:00:49,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742292_1468 (size=12151) 2024-12-08T04:00:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T04:00:49,457 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:49,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-08T04:00:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T04:00:49,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:49,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:49,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T04:00:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733630509587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43676 deadline: 1733630509587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,594 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:49,594 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in 5f0fa7839174f51942b3285bec09c3ac of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:00:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630509597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T04:00:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:49,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630509607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630509607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:49,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/4762c7143f6e410783564c29352df8bf 2024-12-08T04:00:49,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/16a9e92cc0224d7aa87585d2665400e9 is 50, key is test_row_0/C:col10/1733630448580/Put/seqid=0 2024-12-08T04:00:49,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742293_1469 (size=12151) 2024-12-08T04:00:49,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T04:00:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:49,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T04:00:49,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T04:00:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:49,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:50,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T04:00:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:50,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:50,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:50,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:50,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T04:00:50,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630510109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:50,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630510128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630510128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:50,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/16a9e92cc0224d7aa87585d2665400e9 2024-12-08T04:00:50,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/e4adedeaf3e04dffa111af3b12014952 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e4adedeaf3e04dffa111af3b12014952 2024-12-08T04:00:50,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e4adedeaf3e04dffa111af3b12014952, entries=200, sequenceid=221, filesize=14.2 K 2024-12-08T04:00:50,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/4762c7143f6e410783564c29352df8bf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4762c7143f6e410783564c29352df8bf 2024-12-08T04:00:50,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4762c7143f6e410783564c29352df8bf, entries=150, sequenceid=221, filesize=11.9 K 2024-12-08T04:00:50,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/16a9e92cc0224d7aa87585d2665400e9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/16a9e92cc0224d7aa87585d2665400e9 2024-12-08T04:00:50,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/16a9e92cc0224d7aa87585d2665400e9, entries=150, sequenceid=221, filesize=11.9 K 2024-12-08T04:00:50,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5f0fa7839174f51942b3285bec09c3ac in 1283ms, sequenceid=221, compaction requested=true 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:50,195 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T04:00:50,195 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:50,196 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58506 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:50,196 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:50,196 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:50,196 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/db8a142b5a714cb3a0ee19cf79b00730, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/48a6f589bc29469aa668e670442b767f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/c02ef0b28b494bbdad9534fcf75c268e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e4adedeaf3e04dffa111af3b12014952] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=57.1 K 2024-12-08T04:00:50,196 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:50,196 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:50,197 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:50,197 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/84662480ba8245f5b51972f558a20da0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/dc3626b79e764929921b963a498aafaf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/949f51dae2b74d5dbdd1e34bb8f635e5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4762c7143f6e410783564c29352df8bf] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=47.8 K 2024-12-08T04:00:50,197 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting db8a142b5a714cb3a0ee19cf79b00730, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733630446528 2024-12-08T04:00:50,197 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84662480ba8245f5b51972f558a20da0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733630446528 2024-12-08T04:00:50,197 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a6f589bc29469aa668e670442b767f, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733630447253 2024-12-08T04:00:50,197 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc3626b79e764929921b963a498aafaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733630447307 2024-12-08T04:00:50,198 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c02ef0b28b494bbdad9534fcf75c268e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733630447352 2024-12-08T04:00:50,198 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 949f51dae2b74d5dbdd1e34bb8f635e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733630447352 2024-12-08T04:00:50,198 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e4adedeaf3e04dffa111af3b12014952, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733630448572 2024-12-08T04:00:50,199 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4762c7143f6e410783564c29352df8bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733630448572 2024-12-08T04:00:50,222 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:50,222 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/bb85b85f9da340b896b6b7f58b8bdb49 is 50, key is test_row_0/B:col10/1733630448580/Put/seqid=0 2024-12-08T04:00:50,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T04:00:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:50,224 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T04:00:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:50,237 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:50,237 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/3a3062cc3f0e4808b450e3556a2c2e7e is 50, key is test_row_0/A:col10/1733630448580/Put/seqid=0 2024-12-08T04:00:50,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742294_1470 (size=12629) 2024-12-08T04:00:50,264 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/bb85b85f9da340b896b6b7f58b8bdb49 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bb85b85f9da340b896b6b7f58b8bdb49 2024-12-08T04:00:50,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/534363ac1d1247698fc60e1fbb5115cf is 50, key is test_row_0/A:col10/1733630448937/Put/seqid=0 2024-12-08T04:00:50,271 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into bb85b85f9da340b896b6b7f58b8bdb49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:50,271 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:50,271 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=12, startTime=1733630450195; duration=0sec 2024-12-08T04:00:50,271 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:50,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:50,272 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:00:50,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:00:50,274 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:50,274 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:50,274 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b2ca9d663e0b4c929f6efa73965b5a14, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/636d7002119e4bc69dd9ddb314aa69eb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/12abdf787d5548efa1b9c580a08b2a54, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/16a9e92cc0224d7aa87585d2665400e9] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=47.8 K 2024-12-08T04:00:50,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ca9d663e0b4c929f6efa73965b5a14, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733630446528 2024-12-08T04:00:50,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636d7002119e4bc69dd9ddb314aa69eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733630447307 2024-12-08T04:00:50,276 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12abdf787d5548efa1b9c580a08b2a54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733630447352 2024-12-08T04:00:50,277 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16a9e92cc0224d7aa87585d2665400e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733630448572 2024-12-08T04:00:50,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742295_1471 (size=12629) 2024-12-08T04:00:50,300 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/3a3062cc3f0e4808b450e3556a2c2e7e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/3a3062cc3f0e4808b450e3556a2c2e7e 2024-12-08T04:00:50,307 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:50,307 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/94ee36e8933a44508634e365d0768580 is 50, key is test_row_0/C:col10/1733630448580/Put/seqid=0 2024-12-08T04:00:50,311 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into 3a3062cc3f0e4808b450e3556a2c2e7e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:50,311 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:50,311 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=12, startTime=1733630450195; duration=0sec 2024-12-08T04:00:50,311 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:50,311 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:50,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742296_1472 (size=12151) 2024-12-08T04:00:50,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742297_1473 (size=12629) 2024-12-08T04:00:50,351 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/94ee36e8933a44508634e365d0768580 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/94ee36e8933a44508634e365d0768580 2024-12-08T04:00:50,356 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into 94ee36e8933a44508634e365d0768580(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:50,356 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:50,356 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=12, startTime=1733630450195; duration=0sec 2024-12-08T04:00:50,356 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:50,356 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T04:00:50,716 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/534363ac1d1247698fc60e1fbb5115cf 2024-12-08T04:00:50,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/63f2c03639d648e28ea540329aabc582 is 50, key is test_row_0/B:col10/1733630448937/Put/seqid=0 2024-12-08T04:00:50,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742298_1474 (size=12151) 2024-12-08T04:00:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:51,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:51,145 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/63f2c03639d648e28ea540329aabc582 2024-12-08T04:00:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/8068f1d1e47c4838946b79bf7d9f871d is 50, key is test_row_0/C:col10/1733630448937/Put/seqid=0 2024-12-08T04:00:51,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742299_1475 (size=12151) 2024-12-08T04:00:51,167 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/8068f1d1e47c4838946b79bf7d9f871d 2024-12-08T04:00:51,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/534363ac1d1247698fc60e1fbb5115cf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/534363ac1d1247698fc60e1fbb5115cf 2024-12-08T04:00:51,182 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/534363ac1d1247698fc60e1fbb5115cf, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T04:00:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/63f2c03639d648e28ea540329aabc582 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/63f2c03639d648e28ea540329aabc582 2024-12-08T04:00:51,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630511171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630511172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630511174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,187 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/63f2c03639d648e28ea540329aabc582, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T04:00:51,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/8068f1d1e47c4838946b79bf7d9f871d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/8068f1d1e47c4838946b79bf7d9f871d 2024-12-08T04:00:51,193 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/8068f1d1e47c4838946b79bf7d9f871d, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T04:00:51,194 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5f0fa7839174f51942b3285bec09c3ac in 970ms, sequenceid=236, compaction requested=false 2024-12-08T04:00:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-08T04:00:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-08T04:00:51,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-08T04:00:51,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-12-08T04:00:51,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7420 sec 2024-12-08T04:00:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:51,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T04:00:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:51,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/4fe71dc672f24f3abacce290fd9f1a64 is 50, key is test_row_0/A:col10/1733630451292/Put/seqid=0 2024-12-08T04:00:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742300_1476 (size=12201) 2024-12-08T04:00:51,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630511329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630511329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630511331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630511438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630511447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630511447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T04:00:51,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:51,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-08T04:00:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T04:00:51,619 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:51,620 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:51,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:51,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630511657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630511667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630511669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/4fe71dc672f24f3abacce290fd9f1a64 2024-12-08T04:00:51,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/4191f02f74544b88bb1b270726e8ece3 is 50, key is test_row_0/B:col10/1733630451292/Put/seqid=0 2024-12-08T04:00:51,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742301_1477 (size=12201) 2024-12-08T04:00:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T04:00:51,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:51,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:51,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:51,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:51,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T04:00:51,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630511967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630511977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:51,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630511987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:52,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/4191f02f74544b88bb1b270726e8ece3 2024-12-08T04:00:52,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/28c6468acd124c998e73c4d44c4f9b71 is 50, key is test_row_0/C:col10/1733630451292/Put/seqid=0 2024-12-08T04:00:52,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742302_1478 (size=12201) 2024-12-08T04:00:52,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T04:00:52,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630512477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:52,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:52,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:52,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:52,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630512487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630512507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:52,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/28c6468acd124c998e73c4d44c4f9b71 2024-12-08T04:00:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/4fe71dc672f24f3abacce290fd9f1a64 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4fe71dc672f24f3abacce290fd9f1a64 2024-12-08T04:00:52,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4fe71dc672f24f3abacce290fd9f1a64, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T04:00:52,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/4191f02f74544b88bb1b270726e8ece3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4191f02f74544b88bb1b270726e8ece3 2024-12-08T04:00:52,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4191f02f74544b88bb1b270726e8ece3, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T04:00:52,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/28c6468acd124c998e73c4d44c4f9b71 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/28c6468acd124c998e73c4d44c4f9b71 2024-12-08T04:00:52,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:52,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/28c6468acd124c998e73c4d44c4f9b71, entries=150, sequenceid=257, filesize=11.9 K 2024-12-08T04:00:52,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 5f0fa7839174f51942b3285bec09c3ac in 1247ms, sequenceid=257, compaction requested=true 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:52,540 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:52,540 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:52,542 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:52,542 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:52,542 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,542 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/3a3062cc3f0e4808b450e3556a2c2e7e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/534363ac1d1247698fc60e1fbb5115cf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4fe71dc672f24f3abacce290fd9f1a64] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.1 K 2024-12-08T04:00:52,542 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:52,542 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:52,542 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,542 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bb85b85f9da340b896b6b7f58b8bdb49, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/63f2c03639d648e28ea540329aabc582, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4191f02f74544b88bb1b270726e8ece3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.1 K 2024-12-08T04:00:52,542 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a3062cc3f0e4808b450e3556a2c2e7e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733630448572 2024-12-08T04:00:52,543 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting bb85b85f9da340b896b6b7f58b8bdb49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733630448572 2024-12-08T04:00:52,543 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f2c03639d648e28ea540329aabc582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733630448937 2024-12-08T04:00:52,543 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534363ac1d1247698fc60e1fbb5115cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733630448937 2024-12-08T04:00:52,543 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4191f02f74544b88bb1b270726e8ece3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630451163 2024-12-08T04:00:52,544 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe71dc672f24f3abacce290fd9f1a64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630451163 2024-12-08T04:00:52,550 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:52,551 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/026c36d1a16a47129b1d82cd2d109faf is 50, key is test_row_0/B:col10/1733630451292/Put/seqid=0 2024-12-08T04:00:52,553 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:52,553 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/1373a0f3121b4953a2daca89e37cfff8 is 50, key is test_row_0/A:col10/1733630451292/Put/seqid=0 2024-12-08T04:00:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742304_1480 (size=12781) 2024-12-08T04:00:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742303_1479 (size=12781) 2024-12-08T04:00:52,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,688 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/eb22b1b900c24f0490a871ba6be7bd45 is 50, key is test_row_0/A:col10/1733630451328/Put/seqid=0 2024-12-08T04:00:52,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742305_1481 (size=12301) 2024-12-08T04:00:52,697 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/eb22b1b900c24f0490a871ba6be7bd45 2024-12-08T04:00:52,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/95ce8d787ad440a798337d6b214faa6b is 50, key is test_row_0/B:col10/1733630451328/Put/seqid=0 2024-12-08T04:00:52,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742306_1482 (size=12301) 2024-12-08T04:00:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T04:00:52,988 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/026c36d1a16a47129b1d82cd2d109faf as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/026c36d1a16a47129b1d82cd2d109faf 2024-12-08T04:00:52,991 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into 026c36d1a16a47129b1d82cd2d109faf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:52,992 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:52,992 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=13, startTime=1733630452540; duration=0sec 2024-12-08T04:00:52,992 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:52,992 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:52,992 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:52,992 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:52,992 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:52,993 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:52,993 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/94ee36e8933a44508634e365d0768580, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/8068f1d1e47c4838946b79bf7d9f871d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/28c6468acd124c998e73c4d44c4f9b71] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.1 K 2024-12-08T04:00:52,993 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ee36e8933a44508634e365d0768580, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733630448572 2024-12-08T04:00:52,993 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8068f1d1e47c4838946b79bf7d9f871d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733630448937 2024-12-08T04:00:52,993 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c6468acd124c998e73c4d44c4f9b71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630451163 2024-12-08T04:00:52,997 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/1373a0f3121b4953a2daca89e37cfff8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1373a0f3121b4953a2daca89e37cfff8 2024-12-08T04:00:53,002 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into 1373a0f3121b4953a2daca89e37cfff8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:53,002 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:53,002 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=13, startTime=1733630452540; duration=0sec 2024-12-08T04:00:53,002 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:53,002 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:53,004 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:53,005 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/66d57a27f4514ca785f79b71958f29a3 is 50, key is test_row_0/C:col10/1733630451292/Put/seqid=0 2024-12-08T04:00:53,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742307_1483 (size=12781) 2024-12-08T04:00:53,121 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/95ce8d787ad440a798337d6b214faa6b 2024-12-08T04:00:53,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/049232b8db39491c8e8e15fd4dfd89d1 is 50, key is test_row_0/C:col10/1733630451328/Put/seqid=0 2024-12-08T04:00:53,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742308_1484 (size=12301) 2024-12-08T04:00:53,144 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/049232b8db39491c8e8e15fd4dfd89d1 2024-12-08T04:00:53,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/eb22b1b900c24f0490a871ba6be7bd45 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/eb22b1b900c24f0490a871ba6be7bd45 2024-12-08T04:00:53,162 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/eb22b1b900c24f0490a871ba6be7bd45, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T04:00:53,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/95ce8d787ad440a798337d6b214faa6b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/95ce8d787ad440a798337d6b214faa6b 2024-12-08T04:00:53,170 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/95ce8d787ad440a798337d6b214faa6b, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T04:00:53,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/049232b8db39491c8e8e15fd4dfd89d1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/049232b8db39491c8e8e15fd4dfd89d1 2024-12-08T04:00:53,176 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/049232b8db39491c8e8e15fd4dfd89d1, entries=150, sequenceid=272, filesize=12.0 K 2024-12-08T04:00:53,176 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 5f0fa7839174f51942b3285bec09c3ac in 488ms, sequenceid=272, compaction requested=false 2024-12-08T04:00:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-08T04:00:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-08T04:00:53,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-08T04:00:53,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5580 sec 2024-12-08T04:00:53,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5660 sec 2024-12-08T04:00:53,414 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/66d57a27f4514ca785f79b71958f29a3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/66d57a27f4514ca785f79b71958f29a3 2024-12-08T04:00:53,419 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into 66d57a27f4514ca785f79b71958f29a3(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:53,419 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:53,419 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=13, startTime=1733630452540; duration=0sec 2024-12-08T04:00:53,419 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:53,419 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:53,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:00:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:53,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/7c2bf64cd7264a4fb97cfa39ea4e95a7 is 50, key is test_row_0/A:col10/1733630453508/Put/seqid=0 2024-12-08T04:00:53,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742309_1485 (size=14741) 2024-12-08T04:00:53,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630453533, max timestamp=1733630453547Num uniques 859; 2024-12-08T04:00:53,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:53,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630453533, max timestamp=1733630453547Num uniques 859; 2024-12-08T04:00:53,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:53,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630453533, max timestamp=1733630453547Num uniques 859; 2024-12-08T04:00:53,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630513579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630513586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630513587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630513697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630513697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630513697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T04:00:53,767 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:53,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-08T04:00:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T04:00:53,769 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:53,769 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:53,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T04:00:53,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630513907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630513907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630513907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:53,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/7c2bf64cd7264a4fb97cfa39ea4e95a7 2024-12-08T04:00:53,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:53,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:53,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/72285deffb604ff8ae51f5041f04e5a6 is 50, key is test_row_0/B:col10/1733630453508/Put/seqid=0 2024-12-08T04:00:53,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742310_1486 (size=12301) 2024-12-08T04:00:54,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T04:00:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630514216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:54,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630514227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630514227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:54,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/72285deffb604ff8ae51f5041f04e5a6 2024-12-08T04:00:54,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e37c0d87313248de8461f7cc57a4daec is 50, key is test_row_0/C:col10/1733630453508/Put/seqid=0 2024-12-08T04:00:54,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742311_1487 (size=12301) 2024-12-08T04:00:54,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T04:00:54,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:54,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630514727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:54,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e37c0d87313248de8461f7cc57a4daec 2024-12-08T04:00:54,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/7c2bf64cd7264a4fb97cfa39ea4e95a7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/7c2bf64cd7264a4fb97cfa39ea4e95a7 2024-12-08T04:00:54,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/7c2bf64cd7264a4fb97cfa39ea4e95a7, entries=200, sequenceid=285, filesize=14.4 K 2024-12-08T04:00:54,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/72285deffb604ff8ae51f5041f04e5a6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/72285deffb604ff8ae51f5041f04e5a6 2024-12-08T04:00:54,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630514747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630514747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:54,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:54,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/72285deffb604ff8ae51f5041f04e5a6, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T04:00:54,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e37c0d87313248de8461f7cc57a4daec as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e37c0d87313248de8461f7cc57a4daec 2024-12-08T04:00:54,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e37c0d87313248de8461f7cc57a4daec, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T04:00:54,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5f0fa7839174f51942b3285bec09c3ac in 1251ms, sequenceid=285, compaction requested=true 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:54,760 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T04:00:54,761 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:54,762 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:54,762 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:54,762 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,762 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1373a0f3121b4953a2daca89e37cfff8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/eb22b1b900c24f0490a871ba6be7bd45, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/7c2bf64cd7264a4fb97cfa39ea4e95a7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=38.9 K 2024-12-08T04:00:54,762 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1373a0f3121b4953a2daca89e37cfff8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630451163 2024-12-08T04:00:54,763 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:54,763 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:54,763 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,763 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/026c36d1a16a47129b1d82cd2d109faf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/95ce8d787ad440a798337d6b214faa6b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/72285deffb604ff8ae51f5041f04e5a6] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.5 K 2024-12-08T04:00:54,763 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb22b1b900c24f0490a871ba6be7bd45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733630451316 2024-12-08T04:00:54,764 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 026c36d1a16a47129b1d82cd2d109faf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630451163 2024-12-08T04:00:54,764 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c2bf64cd7264a4fb97cfa39ea4e95a7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733630453498 2024-12-08T04:00:54,764 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ce8d787ad440a798337d6b214faa6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733630451316 2024-12-08T04:00:54,765 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 72285deffb604ff8ae51f5041f04e5a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733630453498 2024-12-08T04:00:54,771 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#417 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:54,771 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/46080f4b02be474d926de92957bbde80 is 50, key is test_row_0/B:col10/1733630453508/Put/seqid=0 2024-12-08T04:00:54,771 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:54,771 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/92f84cf403494e2d9e735ad8e253f0e6 is 50, key is test_row_0/A:col10/1733630453508/Put/seqid=0 2024-12-08T04:00:54,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742313_1489 (size=12983) 2024-12-08T04:00:54,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742312_1488 (size=12983) 2024-12-08T04:00:54,805 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/46080f4b02be474d926de92957bbde80 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/46080f4b02be474d926de92957bbde80 2024-12-08T04:00:54,810 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into 46080f4b02be474d926de92957bbde80(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:54,810 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:54,810 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=13, startTime=1733630454760; duration=0sec 2024-12-08T04:00:54,810 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:54,810 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:54,810 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:54,811 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:54,811 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:54,811 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,811 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/66d57a27f4514ca785f79b71958f29a3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/049232b8db39491c8e8e15fd4dfd89d1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e37c0d87313248de8461f7cc57a4daec] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.5 K 2024-12-08T04:00:54,812 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d57a27f4514ca785f79b71958f29a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733630451163 2024-12-08T04:00:54,812 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 049232b8db39491c8e8e15fd4dfd89d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733630451316 2024-12-08T04:00:54,812 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e37c0d87313248de8461f7cc57a4daec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733630453498 2024-12-08T04:00:54,819 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:54,820 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e95ff1bb15cc4512ad71ee7b5df4f738 is 50, key is test_row_0/C:col10/1733630453508/Put/seqid=0 2024-12-08T04:00:54,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742314_1490 (size=12983) 2024-12-08T04:00:54,839 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e95ff1bb15cc4512ad71ee7b5df4f738 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e95ff1bb15cc4512ad71ee7b5df4f738 2024-12-08T04:00:54,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-08T04:00:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:54,839 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T04:00:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:54,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/393c0acacd3b42ecb2dc23757c48816c is 50, key is test_row_0/A:col10/1733630453576/Put/seqid=0 2024-12-08T04:00:54,849 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into e95ff1bb15cc4512ad71ee7b5df4f738(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:54,849 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:54,849 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=13, startTime=1733630454760; duration=0sec 2024-12-08T04:00:54,849 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:54,849 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:54,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742315_1491 (size=12301) 2024-12-08T04:00:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/393c0acacd3b42ecb2dc23757c48816c 2024-12-08T04:00:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/048758677fdd44c093714ffe4107ac18 is 50, key is test_row_0/B:col10/1733630453576/Put/seqid=0 2024-12-08T04:00:54,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742316_1492 (size=12301) 2024-12-08T04:00:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T04:00:55,190 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/92f84cf403494e2d9e735ad8e253f0e6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/92f84cf403494e2d9e735ad8e253f0e6 2024-12-08T04:00:55,193 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into 92f84cf403494e2d9e735ad8e253f0e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:55,193 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:55,193 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=13, startTime=1733630454760; duration=0sec 2024-12-08T04:00:55,193 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:55,193 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:55,290 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/048758677fdd44c093714ffe4107ac18 2024-12-08T04:00:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7 is 50, key is test_row_0/C:col10/1733630453576/Put/seqid=0 2024-12-08T04:00:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742317_1493 (size=12301) 2024-12-08T04:00:55,330 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7 2024-12-08T04:00:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/393c0acacd3b42ecb2dc23757c48816c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/393c0acacd3b42ecb2dc23757c48816c 2024-12-08T04:00:55,352 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/393c0acacd3b42ecb2dc23757c48816c, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T04:00:55,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/048758677fdd44c093714ffe4107ac18 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/048758677fdd44c093714ffe4107ac18 2024-12-08T04:00:55,357 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/048758677fdd44c093714ffe4107ac18, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T04:00:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7 2024-12-08T04:00:55,366 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7, entries=150, sequenceid=322, filesize=12.0 K 2024-12-08T04:00:55,369 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 5f0fa7839174f51942b3285bec09c3ac in 530ms, sequenceid=322, compaction requested=false 2024-12-08T04:00:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-08T04:00:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-08T04:00:55,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-08T04:00:55,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6030 sec 2024-12-08T04:00:55,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.6100 sec 2024-12-08T04:00:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:55,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:00:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:55,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/a4a5b209d93a4d7e9234d474c5bc1762 is 50, key is test_row_0/A:col10/1733630455764/Put/seqid=0 2024-12-08T04:00:55,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742318_1494 (size=14741) 2024-12-08T04:00:55,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630455808, max timestamp=1733630455820Num uniques 859; 2024-12-08T04:00:55,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:55,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630455808, max timestamp=1733630455820Num uniques 859; 2024-12-08T04:00:55,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:55,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630455808, max timestamp=1733630455820Num uniques 859; 2024-12-08T04:00:55,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:55,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630515862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:55,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:55,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630515867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:55,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630515871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-08T04:00:55,917 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:55,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-08T04:00:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T04:00:55,919 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:55,919 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:55,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:55,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630515977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630515987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630515988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T04:00:56,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T04:00:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/a4a5b209d93a4d7e9234d474c5bc1762 2024-12-08T04:00:56,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/24b9d3191125410699ae1169414c28b7 is 50, key is test_row_0/B:col10/1733630455764/Put/seqid=0 2024-12-08T04:00:56,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742319_1495 (size=12301) 2024-12-08T04:00:56,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/24b9d3191125410699ae1169414c28b7 2024-12-08T04:00:56,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/87ac7b075d034bc896e65a9cc82522f6 is 50, key is test_row_0/C:col10/1733630455764/Put/seqid=0 2024-12-08T04:00:56,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742320_1496 (size=12301) 2024-12-08T04:00:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630516197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630516207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630516207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T04:00:56,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:56,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T04:00:56,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T04:00:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630516519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630516527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630516530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T04:00:56,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T04:00:56,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:56,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:56,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/87ac7b075d034bc896e65a9cc82522f6 2024-12-08T04:00:56,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/a4a5b209d93a4d7e9234d474c5bc1762 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/a4a5b209d93a4d7e9234d474c5bc1762 2024-12-08T04:00:56,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/a4a5b209d93a4d7e9234d474c5bc1762, entries=200, sequenceid=334, filesize=14.4 K 2024-12-08T04:00:56,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/24b9d3191125410699ae1169414c28b7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/24b9d3191125410699ae1169414c28b7 2024-12-08T04:00:56,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/24b9d3191125410699ae1169414c28b7, entries=150, sequenceid=334, filesize=12.0 K 2024-12-08T04:00:56,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/87ac7b075d034bc896e65a9cc82522f6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/87ac7b075d034bc896e65a9cc82522f6 2024-12-08T04:00:56,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/87ac7b075d034bc896e65a9cc82522f6, entries=150, sequenceid=334, filesize=12.0 K 2024-12-08T04:00:56,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5f0fa7839174f51942b3285bec09c3ac in 877ms, sequenceid=334, compaction requested=true 2024-12-08T04:00:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:56,642 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:56,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:56,643 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:56,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:56,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:56,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:56,643 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:56,643 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:56,644 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,644 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,644 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/46080f4b02be474d926de92957bbde80, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/048758677fdd44c093714ffe4107ac18, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/24b9d3191125410699ae1169414c28b7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.7 K 2024-12-08T04:00:56,644 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/92f84cf403494e2d9e735ad8e253f0e6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/393c0acacd3b42ecb2dc23757c48816c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/a4a5b209d93a4d7e9234d474c5bc1762] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=39.1 K 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 46080f4b02be474d926de92957bbde80, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733630453498 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f84cf403494e2d9e735ad8e253f0e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733630453498 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 393c0acacd3b42ecb2dc23757c48816c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630453576 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 048758677fdd44c093714ffe4107ac18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630453576 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a5b209d93a4d7e9234d474c5bc1762, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733630455747 2024-12-08T04:00:56,644 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b9d3191125410699ae1169414c28b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733630455759 2024-12-08T04:00:56,654 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#425 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:56,654 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/e8847ed215244f429d2d505f9c7afd58 is 50, key is test_row_0/A:col10/1733630455764/Put/seqid=0 2024-12-08T04:00:56,656 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:56,657 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/7bbe96f8f09942849e9d72b5670581f8 is 50, key is test_row_0/B:col10/1733630455764/Put/seqid=0 2024-12-08T04:00:56,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742321_1497 (size=13085) 2024-12-08T04:00:56,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742322_1498 (size=13085) 2024-12-08T04:00:56,669 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/7bbe96f8f09942849e9d72b5670581f8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/7bbe96f8f09942849e9d72b5670581f8 2024-12-08T04:00:56,672 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into 7bbe96f8f09942849e9d72b5670581f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:56,672 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:56,672 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=13, startTime=1733630456643; duration=0sec 2024-12-08T04:00:56,672 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:56,672 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:56,672 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:56,673 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:56,673 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:56,673 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,673 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e95ff1bb15cc4512ad71ee7b5df4f738, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/87ac7b075d034bc896e65a9cc82522f6] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.7 K 2024-12-08T04:00:56,673 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e95ff1bb15cc4512ad71ee7b5df4f738, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733630453498 2024-12-08T04:00:56,674 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e1bf2cee1b7c44f7bd3ddf5bf7794de7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733630453576 2024-12-08T04:00:56,674 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ac7b075d034bc896e65a9cc82522f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733630455759 2024-12-08T04:00:56,680 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:56,681 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/23b3ee37dfa34097ba491a17596d07ca is 50, key is test_row_0/C:col10/1733630455764/Put/seqid=0 2024-12-08T04:00:56,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742323_1499 (size=13085) 2024-12-08T04:00:56,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:56,703 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:56,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:56,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/921610f95647482fb6ac38b0b068ff6a is 50, key is test_row_0/A:col10/1733630455859/Put/seqid=0 2024-12-08T04:00:56,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742324_1500 (size=12301) 2024-12-08T04:00:56,710 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/921610f95647482fb6ac38b0b068ff6a 2024-12-08T04:00:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/83793398ea5d4ed9b898366d5e2a8084 is 50, key is test_row_0/B:col10/1733630455859/Put/seqid=0 2024-12-08T04:00:56,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742325_1501 (size=12301) 2024-12-08T04:00:56,889 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T04:00:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:57,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T04:00:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630517057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630517058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630517062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,071 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/e8847ed215244f429d2d505f9c7afd58 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e8847ed215244f429d2d505f9c7afd58 2024-12-08T04:00:57,076 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into e8847ed215244f429d2d505f9c7afd58(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:57,076 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:57,076 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=13, startTime=1733630456642; duration=0sec 2024-12-08T04:00:57,076 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:57,076 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:57,091 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/23b3ee37dfa34097ba491a17596d07ca as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/23b3ee37dfa34097ba491a17596d07ca 2024-12-08T04:00:57,102 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into 23b3ee37dfa34097ba491a17596d07ca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:57,102 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:57,102 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=13, startTime=1733630456643; duration=0sec 2024-12-08T04:00:57,102 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:57,102 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:57,124 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/83793398ea5d4ed9b898366d5e2a8084 2024-12-08T04:00:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/da5b0cb4797f4570b79370016b13f07a is 50, key is test_row_0/C:col10/1733630455859/Put/seqid=0 2024-12-08T04:00:57,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742326_1502 (size=12301) 2024-12-08T04:00:57,160 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/da5b0cb4797f4570b79370016b13f07a 2024-12-08T04:00:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/921610f95647482fb6ac38b0b068ff6a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/921610f95647482fb6ac38b0b068ff6a 2024-12-08T04:00:57,178 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/921610f95647482fb6ac38b0b068ff6a, entries=150, sequenceid=371, filesize=12.0 K 2024-12-08T04:00:57,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/83793398ea5d4ed9b898366d5e2a8084 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/83793398ea5d4ed9b898366d5e2a8084 2024-12-08T04:00:57,185 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/83793398ea5d4ed9b898366d5e2a8084, entries=150, sequenceid=371, filesize=12.0 K 2024-12-08T04:00:57,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/da5b0cb4797f4570b79370016b13f07a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/da5b0cb4797f4570b79370016b13f07a 2024-12-08T04:00:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630517177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630517177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630517178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,190 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/da5b0cb4797f4570b79370016b13f07a, entries=150, sequenceid=371, filesize=12.0 K 2024-12-08T04:00:57,191 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5f0fa7839174f51942b3285bec09c3ac in 488ms, sequenceid=371, compaction requested=false 2024-12-08T04:00:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-08T04:00:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-08T04:00:57,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-08T04:00:57,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2730 sec 2024-12-08T04:00:57,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.2770 sec 2024-12-08T04:00:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:57,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T04:00:57,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:57,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:57,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:57,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:57,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:57,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:57,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/59cce8f9bdae4dbba220ae6ee8c79d4e is 50, key is test_row_0/A:col10/1733630457060/Put/seqid=0 2024-12-08T04:00:57,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742327_1503 (size=14741) 2024-12-08T04:00:57,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/59cce8f9bdae4dbba220ae6ee8c79d4e 2024-12-08T04:00:57,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/e274af6e205b4357a46560b0d887e00d is 50, key is test_row_0/B:col10/1733630457060/Put/seqid=0 2024-12-08T04:00:57,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742328_1504 (size=12301) 2024-12-08T04:00:57,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630517444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630457434, max timestamp=1733630457448Num uniques 859; 2024-12-08T04:00:57,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:57,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630457434, max timestamp=1733630457448Num uniques 859; 2024-12-08T04:00:57,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630457434, max timestamp=1733630457448Num uniques 859; 2024-12-08T04:00:57,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:57,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:00:57,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630517497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630517500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630517557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630517617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630517617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630517777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/e274af6e205b4357a46560b0d887e00d 2024-12-08T04:00:57,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/d1630148ece34e5485c48b7299e07093 is 50, key is test_row_0/C:col10/1733630457060/Put/seqid=0 2024-12-08T04:00:57,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742329_1505 (size=12301) 2024-12-08T04:00:57,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/d1630148ece34e5485c48b7299e07093 2024-12-08T04:00:57,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/59cce8f9bdae4dbba220ae6ee8c79d4e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/59cce8f9bdae4dbba220ae6ee8c79d4e 2024-12-08T04:00:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630517837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630517838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:57,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/59cce8f9bdae4dbba220ae6ee8c79d4e, entries=200, sequenceid=384, filesize=14.4 K 2024-12-08T04:00:57,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/e274af6e205b4357a46560b0d887e00d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/e274af6e205b4357a46560b0d887e00d 2024-12-08T04:00:57,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/e274af6e205b4357a46560b0d887e00d, entries=150, sequenceid=384, filesize=12.0 K 2024-12-08T04:00:57,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/d1630148ece34e5485c48b7299e07093 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1630148ece34e5485c48b7299e07093 2024-12-08T04:00:57,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1630148ece34e5485c48b7299e07093, entries=150, sequenceid=384, filesize=12.0 K 2024-12-08T04:00:57,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5f0fa7839174f51942b3285bec09c3ac in 469ms, sequenceid=384, compaction requested=true 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:57,870 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:57,870 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0fa7839174f51942b3285bec09c3ac:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:00:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:57,871 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:57,871 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/A is initiating minor compaction (all files) 2024-12-08T04:00:57,871 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/A in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:57,871 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e8847ed215244f429d2d505f9c7afd58, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/921610f95647482fb6ac38b0b068ff6a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/59cce8f9bdae4dbba220ae6ee8c79d4e] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=39.2 K 2024-12-08T04:00:57,872 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:57,872 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/B is initiating minor compaction (all files) 2024-12-08T04:00:57,872 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/B in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:57,872 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/7bbe96f8f09942849e9d72b5670581f8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/83793398ea5d4ed9b898366d5e2a8084, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/e274af6e205b4357a46560b0d887e00d] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.8 K 2024-12-08T04:00:57,872 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8847ed215244f429d2d505f9c7afd58, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733630455759 2024-12-08T04:00:57,872 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bbe96f8f09942849e9d72b5670581f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733630455759 2024-12-08T04:00:57,873 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 921610f95647482fb6ac38b0b068ff6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733630455859 2024-12-08T04:00:57,873 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 83793398ea5d4ed9b898366d5e2a8084, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733630455859 2024-12-08T04:00:57,874 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e274af6e205b4357a46560b0d887e00d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733630457054 2024-12-08T04:00:57,874 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59cce8f9bdae4dbba220ae6ee8c79d4e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733630457038 2024-12-08T04:00:57,880 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:57,881 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/d11e028889f54eefaf08c5057b666497 is 50, key is test_row_0/B:col10/1733630457060/Put/seqid=0 2024-12-08T04:00:57,889 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:57,889 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d710525d60894d009bbc4ff59f44098d is 50, key is test_row_0/A:col10/1733630457060/Put/seqid=0 2024-12-08T04:00:57,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742330_1506 (size=13187) 2024-12-08T04:00:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742331_1507 (size=13187) 2024-12-08T04:00:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T04:00:58,078 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:00:58,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:00:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-08T04:00:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:00:58,080 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:00:58,081 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:00:58,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:00:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:58,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-08T04:00:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:58,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:00:58,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/1ecfd28f38be41cc83646eaf18a83be9 is 50, key is test_row_0/A:col10/1733630458092/Put/seqid=0 2024-12-08T04:00:58,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742332_1508 (size=14741) 2024-12-08T04:00:58,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630518157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630518163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630518170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:00:58,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630518287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630518287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,315 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/d11e028889f54eefaf08c5057b666497 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/d11e028889f54eefaf08c5057b666497 2024-12-08T04:00:58,319 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d710525d60894d009bbc4ff59f44098d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d710525d60894d009bbc4ff59f44098d 2024-12-08T04:00:58,321 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/B of 5f0fa7839174f51942b3285bec09c3ac into d11e028889f54eefaf08c5057b666497(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:58,321 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:58,321 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/B, priority=13, startTime=1733630457870; duration=0sec 2024-12-08T04:00:58,321 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:00:58,321 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:B 2024-12-08T04:00:58,321 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:00:58,323 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:00:58,324 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): 5f0fa7839174f51942b3285bec09c3ac/C is initiating minor compaction (all files) 2024-12-08T04:00:58,324 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0fa7839174f51942b3285bec09c3ac/C in TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,324 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/23b3ee37dfa34097ba491a17596d07ca, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/da5b0cb4797f4570b79370016b13f07a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1630148ece34e5485c48b7299e07093] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp, totalSize=36.8 K 2024-12-08T04:00:58,324 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b3ee37dfa34097ba491a17596d07ca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733630455759 2024-12-08T04:00:58,325 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting da5b0cb4797f4570b79370016b13f07a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733630455859 2024-12-08T04:00:58,325 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d1630148ece34e5485c48b7299e07093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733630457054 2024-12-08T04:00:58,326 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/A of 5f0fa7839174f51942b3285bec09c3ac into d710525d60894d009bbc4ff59f44098d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:58,326 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:58,326 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/A, priority=13, startTime=1733630457870; duration=0sec 2024-12-08T04:00:58,326 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:58,326 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:A 2024-12-08T04:00:58,336 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0fa7839174f51942b3285bec09c3ac#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:00:58,337 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/fd13da4e3faf46da95c43b7388a8845a is 50, key is test_row_0/C:col10/1733630457060/Put/seqid=0 2024-12-08T04:00:58,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742333_1509 (size=13187) 2024-12-08T04:00:58,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:00:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630518507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/1ecfd28f38be41cc83646eaf18a83be9 2024-12-08T04:00:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630518507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/659ceb0cac8c4bbea65ae28a488df934 is 50, key is test_row_0/B:col10/1733630458092/Put/seqid=0 2024-12-08T04:00:58,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742334_1510 (size=12301) 2024-12-08T04:00:58,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:58,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43686 deadline: 1733630518677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:58,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:58,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:00:58,753 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/fd13da4e3faf46da95c43b7388a8845a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/fd13da4e3faf46da95c43b7388a8845a 2024-12-08T04:00:58,759 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0fa7839174f51942b3285bec09c3ac/C of 5f0fa7839174f51942b3285bec09c3ac into fd13da4e3faf46da95c43b7388a8845a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:00:58,759 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:58,759 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., storeName=5f0fa7839174f51942b3285bec09c3ac/C, priority=13, startTime=1733630457870; duration=0sec 2024-12-08T04:00:58,759 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:00:58,759 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0fa7839174f51942b3285bec09c3ac:C 2024-12-08T04:00:58,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630518827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630518827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:58,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/659ceb0cac8c4bbea65ae28a488df934 2024-12-08T04:00:58,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/b4c0288ceeab4002a45dd28ca455e9f1 is 50, key is test_row_0/C:col10/1733630458092/Put/seqid=0 2024-12-08T04:00:58,962 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-12-08T04:00:58,962 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:58,962 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,962 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:58,964 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-12-08T04:00:58,964 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:58,964 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,964 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:58,966 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-12-08T04:00:58,967 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:58,967 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,967 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:58,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742335_1511 (size=12301) 2024-12-08T04:00:58,972 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-12-08T04:00:58,972 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:58,972 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,972 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:58,974 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-12-08T04:00:58,974 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:58,974 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,974 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:58,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:58,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:58,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:59,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:59,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:00:59,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:59,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:59,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:59,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:59,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:00:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:00:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43600 deadline: 1733630519347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43680 deadline: 1733630519347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:00:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 , the old value is region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:00:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., hostname=32df4b4bb439,39919,1733630308479, seqNum=2 because the exception is null or not the one we care about 2024-12-08T04:00:59,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/b4c0288ceeab4002a45dd28ca455e9f1 2024-12-08T04:00:59,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/1ecfd28f38be41cc83646eaf18a83be9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1ecfd28f38be41cc83646eaf18a83be9 2024-12-08T04:00:59,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1ecfd28f38be41cc83646eaf18a83be9, entries=200, sequenceid=420, filesize=14.4 K 2024-12-08T04:00:59,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/659ceb0cac8c4bbea65ae28a488df934 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/659ceb0cac8c4bbea65ae28a488df934 2024-12-08T04:00:59,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/659ceb0cac8c4bbea65ae28a488df934, entries=150, sequenceid=420, filesize=12.0 K 2024-12-08T04:00:59,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/b4c0288ceeab4002a45dd28ca455e9f1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b4c0288ceeab4002a45dd28ca455e9f1 2024-12-08T04:00:59,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b4c0288ceeab4002a45dd28ca455e9f1, entries=150, sequenceid=420, filesize=12.0 K 2024-12-08T04:00:59,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5f0fa7839174f51942b3285bec09c3ac in 1296ms, sequenceid=420, compaction requested=false 2024-12-08T04:00:59,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:00:59,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-08T04:00:59,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:00:59,457 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:00:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:00:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:00:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:00:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:00:59,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/f3843cd649df4a8d8621b9fe771d2f56 is 50, key is test_row_0/A:col10/1733630458160/Put/seqid=0 2024-12-08T04:00:59,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742336_1512 (size=12301) 2024-12-08T04:00:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:00:59,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. as already flushing 2024-12-08T04:00:59,619 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-12-08T04:00:59,619 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:59,619 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:59,619 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:59,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:59,678 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-12-08T04:00:59,678 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:59,678 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:59,678 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:59,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:59,688 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-12-08T04:00:59,688 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:00:59,688 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:59,689 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:00:59,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:00:59,870 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/f3843cd649df4a8d8621b9fe771d2f56 2024-12-08T04:00:59,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/92aabdb10cd64476aeb45a2d60954038 is 50, key is test_row_0/B:col10/1733630458160/Put/seqid=0 2024-12-08T04:00:59,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742337_1513 (size=12301) 2024-12-08T04:01:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:01:00,280 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/92aabdb10cd64476aeb45a2d60954038 2024-12-08T04:01:00,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/d1164b1b62cf47859a22fe1f25d3f2ee is 50, key is test_row_0/C:col10/1733630458160/Put/seqid=0 2024-12-08T04:01:00,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742338_1514 (size=12301) 2024-12-08T04:01:00,358 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-12-08T04:01:00,358 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-12-08T04:01:00,358 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:00,358 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:00,358 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:00,358 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:00,358 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:00,358 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:00,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:00,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:00,691 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/d1164b1b62cf47859a22fe1f25d3f2ee 2024-12-08T04:01:00,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/f3843cd649df4a8d8621b9fe771d2f56 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f3843cd649df4a8d8621b9fe771d2f56 2024-12-08T04:01:00,698 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f3843cd649df4a8d8621b9fe771d2f56, entries=150, sequenceid=433, filesize=12.0 K 2024-12-08T04:01:00,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/92aabdb10cd64476aeb45a2d60954038 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/92aabdb10cd64476aeb45a2d60954038 2024-12-08T04:01:00,702 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/92aabdb10cd64476aeb45a2d60954038, entries=150, sequenceid=433, filesize=12.0 K 2024-12-08T04:01:00,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/d1164b1b62cf47859a22fe1f25d3f2ee as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1164b1b62cf47859a22fe1f25d3f2ee 2024-12-08T04:01:00,705 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1164b1b62cf47859a22fe1f25d3f2ee, entries=150, sequenceid=433, filesize=12.0 K 2024-12-08T04:01:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for 5f0fa7839174f51942b3285bec09c3ac in 1250ms, sequenceid=433, compaction requested=true 2024-12-08T04:01:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 5f0fa7839174f51942b3285bec09c3ac: 2024-12-08T04:01:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:01:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-08T04:01:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-08T04:01:00,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-08T04:01:00,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6260 sec 2024-12-08T04:01:00,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.6310 sec 2024-12-08T04:01:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-08T04:01:02,248 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1613 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4832 rows 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1609 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4820 rows 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1614 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4840 rows 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1613 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4835 rows 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1611 2024-12-08T04:01:02,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4829 rows 2024-12-08T04:01:02,248 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:02,248 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:02,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:02,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:02,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:02,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T04:01:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T04:01:02,253 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630462253"}]},"ts":"1733630462253"} 2024-12-08T04:01:02,254 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T04:01:02,254 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T04:01:02,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T04:01:02,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, UNASSIGN}] 2024-12-08T04:01:02,258 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, UNASSIGN 2024-12-08T04:01:02,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=5f0fa7839174f51942b3285bec09c3ac, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:02,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, UNASSIGN because future has completed 2024-12-08T04:01:02,262 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T04:01:02,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:01:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T04:01:02,415 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:01:02,415 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T04:01:02,415 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 5f0fa7839174f51942b3285bec09c3ac, disabling compactions & flushes 2024-12-08T04:01:02,416 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:01:02,416 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:01:02,416 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. after waiting 0 ms 2024-12-08T04:01:02,416 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:01:02,416 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 5f0fa7839174f51942b3285bec09c3ac 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T04:01:02,416 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=A 2024-12-08T04:01:02,417 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:02,417 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=B 2024-12-08T04:01:02,417 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:02,417 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5f0fa7839174f51942b3285bec09c3ac, store=C 2024-12-08T04:01:02,417 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:02,424 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af is 50, key is test_row_1/A:col10/1733630460357/Put/seqid=0 2024-12-08T04:01:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742339_1515 (size=9857) 2024-12-08T04:01:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T04:01:02,830 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af 2024-12-08T04:01:02,840 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/2db0ef54daf8452491b11998f59ee685 is 50, key is test_row_1/B:col10/1733630460357/Put/seqid=0 2024-12-08T04:01:02,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742340_1516 (size=9857) 2024-12-08T04:01:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T04:01:03,245 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/2db0ef54daf8452491b11998f59ee685 2024-12-08T04:01:03,258 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/6785d6934fc4488688a685039ef145ca is 50, key is test_row_1/C:col10/1733630460357/Put/seqid=0 2024-12-08T04:01:03,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742341_1517 (size=9857) 2024-12-08T04:01:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T04:01:03,663 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/6785d6934fc4488688a685039ef145ca 2024-12-08T04:01:03,673 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af 2024-12-08T04:01:03,680 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af, entries=100, sequenceid=441, filesize=9.6 K 2024-12-08T04:01:03,681 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/B/2db0ef54daf8452491b11998f59ee685 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/2db0ef54daf8452491b11998f59ee685 2024-12-08T04:01:03,685 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/2db0ef54daf8452491b11998f59ee685, entries=100, sequenceid=441, filesize=9.6 K 2024-12-08T04:01:03,685 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/.tmp/C/6785d6934fc4488688a685039ef145ca as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/6785d6934fc4488688a685039ef145ca 2024-12-08T04:01:03,688 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/6785d6934fc4488688a685039ef145ca, entries=100, sequenceid=441, filesize=9.6 K 2024-12-08T04:01:03,689 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5f0fa7839174f51942b3285bec09c3ac in 1273ms, sequenceid=441, compaction requested=true 2024-12-08T04:01:03,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d96d0ed90e754438ba297b71b9921811, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4d565923545b4596bb7ced9298f51a1f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/49396407358849aa9a93335fb790fad7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/51ecf6f259f44aefb6b3e04bc976aaeb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/031673738e834b6eb83d0abf198717da, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/34231a1328c9408f95067e05a8993ccf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/ec2eff0e705d462c9529b20baba5f242, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/942f4f981daf4665804e81ab5e0acd79, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f4c92b25a27949418e000119a14ba7cd, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/5b9ccf6af0254371b084195bc8b1efd1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/db8a142b5a714cb3a0ee19cf79b00730, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/48a6f589bc29469aa668e670442b767f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/c02ef0b28b494bbdad9534fcf75c268e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e4adedeaf3e04dffa111af3b12014952, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/3a3062cc3f0e4808b450e3556a2c2e7e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/534363ac1d1247698fc60e1fbb5115cf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1373a0f3121b4953a2daca89e37cfff8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4fe71dc672f24f3abacce290fd9f1a64, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/eb22b1b900c24f0490a871ba6be7bd45, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/7c2bf64cd7264a4fb97cfa39ea4e95a7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/92f84cf403494e2d9e735ad8e253f0e6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/393c0acacd3b42ecb2dc23757c48816c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/a4a5b209d93a4d7e9234d474c5bc1762, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e8847ed215244f429d2d505f9c7afd58, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/921610f95647482fb6ac38b0b068ff6a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/59cce8f9bdae4dbba220ae6ee8c79d4e] to archive 2024-12-08T04:01:03,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:01:03,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d96d0ed90e754438ba297b71b9921811 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d96d0ed90e754438ba297b71b9921811 2024-12-08T04:01:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4d565923545b4596bb7ced9298f51a1f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4d565923545b4596bb7ced9298f51a1f 2024-12-08T04:01:03,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/49396407358849aa9a93335fb790fad7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/49396407358849aa9a93335fb790fad7 2024-12-08T04:01:03,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/51ecf6f259f44aefb6b3e04bc976aaeb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/51ecf6f259f44aefb6b3e04bc976aaeb 2024-12-08T04:01:03,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/031673738e834b6eb83d0abf198717da to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/031673738e834b6eb83d0abf198717da 2024-12-08T04:01:03,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/34231a1328c9408f95067e05a8993ccf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/34231a1328c9408f95067e05a8993ccf 2024-12-08T04:01:03,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/ec2eff0e705d462c9529b20baba5f242 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/ec2eff0e705d462c9529b20baba5f242 2024-12-08T04:01:03,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/942f4f981daf4665804e81ab5e0acd79 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/942f4f981daf4665804e81ab5e0acd79 2024-12-08T04:01:03,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f4c92b25a27949418e000119a14ba7cd to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f4c92b25a27949418e000119a14ba7cd 2024-12-08T04:01:03,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/5b9ccf6af0254371b084195bc8b1efd1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/5b9ccf6af0254371b084195bc8b1efd1 2024-12-08T04:01:03,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/db8a142b5a714cb3a0ee19cf79b00730 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/db8a142b5a714cb3a0ee19cf79b00730 2024-12-08T04:01:03,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/48a6f589bc29469aa668e670442b767f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/48a6f589bc29469aa668e670442b767f 2024-12-08T04:01:03,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/c02ef0b28b494bbdad9534fcf75c268e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/c02ef0b28b494bbdad9534fcf75c268e 2024-12-08T04:01:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e4adedeaf3e04dffa111af3b12014952 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e4adedeaf3e04dffa111af3b12014952 2024-12-08T04:01:03,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/3a3062cc3f0e4808b450e3556a2c2e7e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/3a3062cc3f0e4808b450e3556a2c2e7e 2024-12-08T04:01:03,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/534363ac1d1247698fc60e1fbb5115cf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/534363ac1d1247698fc60e1fbb5115cf 2024-12-08T04:01:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1373a0f3121b4953a2daca89e37cfff8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1373a0f3121b4953a2daca89e37cfff8 2024-12-08T04:01:03,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4fe71dc672f24f3abacce290fd9f1a64 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/4fe71dc672f24f3abacce290fd9f1a64 2024-12-08T04:01:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/eb22b1b900c24f0490a871ba6be7bd45 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/eb22b1b900c24f0490a871ba6be7bd45 2024-12-08T04:01:03,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/7c2bf64cd7264a4fb97cfa39ea4e95a7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/7c2bf64cd7264a4fb97cfa39ea4e95a7 2024-12-08T04:01:03,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/92f84cf403494e2d9e735ad8e253f0e6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/92f84cf403494e2d9e735ad8e253f0e6 2024-12-08T04:01:03,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/393c0acacd3b42ecb2dc23757c48816c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/393c0acacd3b42ecb2dc23757c48816c 2024-12-08T04:01:03,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/a4a5b209d93a4d7e9234d474c5bc1762 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/a4a5b209d93a4d7e9234d474c5bc1762 2024-12-08T04:01:03,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e8847ed215244f429d2d505f9c7afd58 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/e8847ed215244f429d2d505f9c7afd58 2024-12-08T04:01:03,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/921610f95647482fb6ac38b0b068ff6a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/921610f95647482fb6ac38b0b068ff6a 2024-12-08T04:01:03,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/59cce8f9bdae4dbba220ae6ee8c79d4e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/59cce8f9bdae4dbba220ae6ee8c79d4e 2024-12-08T04:01:03,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/1092d2ac355844f883de40df4fd43536, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/06e524b01b6145c7bb03a453dc1744b0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9bcb2c1e3f674d4f90346fe33a82d654, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9fe0230840204ab680cc9c68bdb80272, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/20adbb22b29c4fb29b719ea46c477473, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/fcce68535cb64ab8aeb2cdcb563a0632, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/3a77f58faade4c208cb08038a3b28015, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9017d7de912648c4ab398370b826a0dc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/f2ea115d5c04494194eb6956db4c9fbc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/84662480ba8245f5b51972f558a20da0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bf368a9d316f4f94857f848c4128d938, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/dc3626b79e764929921b963a498aafaf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/949f51dae2b74d5dbdd1e34bb8f635e5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bb85b85f9da340b896b6b7f58b8bdb49, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4762c7143f6e410783564c29352df8bf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/63f2c03639d648e28ea540329aabc582, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/026c36d1a16a47129b1d82cd2d109faf, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4191f02f74544b88bb1b270726e8ece3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/95ce8d787ad440a798337d6b214faa6b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/46080f4b02be474d926de92957bbde80, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/72285deffb604ff8ae51f5041f04e5a6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/048758677fdd44c093714ffe4107ac18, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/7bbe96f8f09942849e9d72b5670581f8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/24b9d3191125410699ae1169414c28b7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/83793398ea5d4ed9b898366d5e2a8084, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/e274af6e205b4357a46560b0d887e00d] to archive 2024-12-08T04:01:03,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:01:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/1092d2ac355844f883de40df4fd43536 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/1092d2ac355844f883de40df4fd43536 2024-12-08T04:01:03,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/06e524b01b6145c7bb03a453dc1744b0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/06e524b01b6145c7bb03a453dc1744b0 2024-12-08T04:01:03,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9bcb2c1e3f674d4f90346fe33a82d654 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9bcb2c1e3f674d4f90346fe33a82d654 2024-12-08T04:01:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9fe0230840204ab680cc9c68bdb80272 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9fe0230840204ab680cc9c68bdb80272 2024-12-08T04:01:03,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/20adbb22b29c4fb29b719ea46c477473 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/20adbb22b29c4fb29b719ea46c477473 2024-12-08T04:01:03,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/fcce68535cb64ab8aeb2cdcb563a0632 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/fcce68535cb64ab8aeb2cdcb563a0632 2024-12-08T04:01:03,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/3a77f58faade4c208cb08038a3b28015 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/3a77f58faade4c208cb08038a3b28015 2024-12-08T04:01:03,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9017d7de912648c4ab398370b826a0dc to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/9017d7de912648c4ab398370b826a0dc 2024-12-08T04:01:03,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/f2ea115d5c04494194eb6956db4c9fbc to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/f2ea115d5c04494194eb6956db4c9fbc 2024-12-08T04:01:03,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/84662480ba8245f5b51972f558a20da0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/84662480ba8245f5b51972f558a20da0 2024-12-08T04:01:03,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bf368a9d316f4f94857f848c4128d938 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bf368a9d316f4f94857f848c4128d938 2024-12-08T04:01:03,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/dc3626b79e764929921b963a498aafaf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/dc3626b79e764929921b963a498aafaf 2024-12-08T04:01:03,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/949f51dae2b74d5dbdd1e34bb8f635e5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/949f51dae2b74d5dbdd1e34bb8f635e5 2024-12-08T04:01:03,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bb85b85f9da340b896b6b7f58b8bdb49 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/bb85b85f9da340b896b6b7f58b8bdb49 2024-12-08T04:01:03,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4762c7143f6e410783564c29352df8bf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4762c7143f6e410783564c29352df8bf 2024-12-08T04:01:03,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/63f2c03639d648e28ea540329aabc582 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/63f2c03639d648e28ea540329aabc582 2024-12-08T04:01:03,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/026c36d1a16a47129b1d82cd2d109faf to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/026c36d1a16a47129b1d82cd2d109faf 2024-12-08T04:01:03,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4191f02f74544b88bb1b270726e8ece3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/4191f02f74544b88bb1b270726e8ece3 2024-12-08T04:01:03,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/95ce8d787ad440a798337d6b214faa6b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/95ce8d787ad440a798337d6b214faa6b 2024-12-08T04:01:03,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/46080f4b02be474d926de92957bbde80 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/46080f4b02be474d926de92957bbde80 2024-12-08T04:01:03,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/72285deffb604ff8ae51f5041f04e5a6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/72285deffb604ff8ae51f5041f04e5a6 2024-12-08T04:01:03,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/048758677fdd44c093714ffe4107ac18 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/048758677fdd44c093714ffe4107ac18 2024-12-08T04:01:03,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/7bbe96f8f09942849e9d72b5670581f8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/7bbe96f8f09942849e9d72b5670581f8 2024-12-08T04:01:03,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/24b9d3191125410699ae1169414c28b7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/24b9d3191125410699ae1169414c28b7 2024-12-08T04:01:03,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/83793398ea5d4ed9b898366d5e2a8084 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/83793398ea5d4ed9b898366d5e2a8084 2024-12-08T04:01:03,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/e274af6e205b4357a46560b0d887e00d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/e274af6e205b4357a46560b0d887e00d 2024-12-08T04:01:03,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/32d7d12d7779421db195880826d0290f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/36b134ff7b434146adb1a80b4a93a51d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1d43b7d000e5404c861c2211bcb770f2, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/7010273f8c5a43ecaf628bec8e9e820d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1e880e9a0d51416d9c53fa7bbb8c9ede, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/9770a099367e4a5bb58f83a3920d452e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/027804ae51a243258132c0faba795e5e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f62c483ccfe6478f8a9cb98cd0e8c592, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/4004fa79243741ba9742505fa100d710, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b2ca9d663e0b4c929f6efa73965b5a14, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f256cc1c9c564c1b909b45b364db40e1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/636d7002119e4bc69dd9ddb314aa69eb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/12abdf787d5548efa1b9c580a08b2a54, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/94ee36e8933a44508634e365d0768580, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/16a9e92cc0224d7aa87585d2665400e9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/8068f1d1e47c4838946b79bf7d9f871d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/66d57a27f4514ca785f79b71958f29a3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/28c6468acd124c998e73c4d44c4f9b71, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/049232b8db39491c8e8e15fd4dfd89d1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e95ff1bb15cc4512ad71ee7b5df4f738, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e37c0d87313248de8461f7cc57a4daec, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/23b3ee37dfa34097ba491a17596d07ca, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/87ac7b075d034bc896e65a9cc82522f6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/da5b0cb4797f4570b79370016b13f07a, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1630148ece34e5485c48b7299e07093] to archive 2024-12-08T04:01:03,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:01:03,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/32d7d12d7779421db195880826d0290f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/32d7d12d7779421db195880826d0290f 2024-12-08T04:01:03,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/36b134ff7b434146adb1a80b4a93a51d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/36b134ff7b434146adb1a80b4a93a51d 2024-12-08T04:01:03,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1d43b7d000e5404c861c2211bcb770f2 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1d43b7d000e5404c861c2211bcb770f2 2024-12-08T04:01:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/7010273f8c5a43ecaf628bec8e9e820d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/7010273f8c5a43ecaf628bec8e9e820d 2024-12-08T04:01:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1e880e9a0d51416d9c53fa7bbb8c9ede to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/1e880e9a0d51416d9c53fa7bbb8c9ede 2024-12-08T04:01:03,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/9770a099367e4a5bb58f83a3920d452e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/9770a099367e4a5bb58f83a3920d452e 2024-12-08T04:01:03,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/027804ae51a243258132c0faba795e5e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/027804ae51a243258132c0faba795e5e 2024-12-08T04:01:03,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f62c483ccfe6478f8a9cb98cd0e8c592 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f62c483ccfe6478f8a9cb98cd0e8c592 2024-12-08T04:01:03,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/4004fa79243741ba9742505fa100d710 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/4004fa79243741ba9742505fa100d710 2024-12-08T04:01:03,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b2ca9d663e0b4c929f6efa73965b5a14 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b2ca9d663e0b4c929f6efa73965b5a14 2024-12-08T04:01:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f256cc1c9c564c1b909b45b364db40e1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/f256cc1c9c564c1b909b45b364db40e1 2024-12-08T04:01:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/636d7002119e4bc69dd9ddb314aa69eb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/636d7002119e4bc69dd9ddb314aa69eb 2024-12-08T04:01:03,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/12abdf787d5548efa1b9c580a08b2a54 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/12abdf787d5548efa1b9c580a08b2a54 2024-12-08T04:01:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/94ee36e8933a44508634e365d0768580 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/94ee36e8933a44508634e365d0768580 2024-12-08T04:01:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/16a9e92cc0224d7aa87585d2665400e9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/16a9e92cc0224d7aa87585d2665400e9 2024-12-08T04:01:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/8068f1d1e47c4838946b79bf7d9f871d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/8068f1d1e47c4838946b79bf7d9f871d 2024-12-08T04:01:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/66d57a27f4514ca785f79b71958f29a3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/66d57a27f4514ca785f79b71958f29a3 2024-12-08T04:01:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/28c6468acd124c998e73c4d44c4f9b71 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/28c6468acd124c998e73c4d44c4f9b71 2024-12-08T04:01:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/049232b8db39491c8e8e15fd4dfd89d1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/049232b8db39491c8e8e15fd4dfd89d1 2024-12-08T04:01:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e95ff1bb15cc4512ad71ee7b5df4f738 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e95ff1bb15cc4512ad71ee7b5df4f738 2024-12-08T04:01:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e37c0d87313248de8461f7cc57a4daec to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e37c0d87313248de8461f7cc57a4daec 2024-12-08T04:01:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/e1bf2cee1b7c44f7bd3ddf5bf7794de7 2024-12-08T04:01:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/23b3ee37dfa34097ba491a17596d07ca to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/23b3ee37dfa34097ba491a17596d07ca 2024-12-08T04:01:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/87ac7b075d034bc896e65a9cc82522f6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/87ac7b075d034bc896e65a9cc82522f6 2024-12-08T04:01:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/da5b0cb4797f4570b79370016b13f07a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/da5b0cb4797f4570b79370016b13f07a 2024-12-08T04:01:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1630148ece34e5485c48b7299e07093 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1630148ece34e5485c48b7299e07093 2024-12-08T04:01:03,748 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/recovered.edits/444.seqid, newMaxSeqId=444, maxSeqId=1 2024-12-08T04:01:03,748 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. 2024-12-08T04:01:03,748 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 5f0fa7839174f51942b3285bec09c3ac: Waiting for close lock at 1733630462415Running coprocessor pre-close hooks at 1733630462415Disabling compacts and flushes for region at 1733630462415Disabling writes for close at 1733630462416 (+1 ms)Obtaining lock to block concurrent updates at 1733630462416Preparing flush snapshotting stores in 5f0fa7839174f51942b3285bec09c3ac at 1733630462416Finished memstore snapshotting TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733630462417 (+1 ms)Flushing stores of TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac. at 1733630462418 (+1 ms)Flushing 5f0fa7839174f51942b3285bec09c3ac/A: creating writer at 1733630462418Flushing 5f0fa7839174f51942b3285bec09c3ac/A: appending metadata at 1733630462423 (+5 ms)Flushing 5f0fa7839174f51942b3285bec09c3ac/A: closing flushed file at 1733630462423Flushing 5f0fa7839174f51942b3285bec09c3ac/B: creating writer at 1733630462836 (+413 ms)Flushing 5f0fa7839174f51942b3285bec09c3ac/B: appending metadata at 1733630462839 (+3 ms)Flushing 5f0fa7839174f51942b3285bec09c3ac/B: closing flushed file at 1733630462839Flushing 5f0fa7839174f51942b3285bec09c3ac/C: creating writer at 1733630463253 (+414 ms)Flushing 5f0fa7839174f51942b3285bec09c3ac/C: appending metadata at 1733630463258 (+5 ms)Flushing 5f0fa7839174f51942b3285bec09c3ac/C: closing flushed file at 1733630463258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a517740: reopening flushed file at 1733630463672 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35a77f46: reopening flushed file at 1733630463680 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f6e588d: reopening flushed file at 1733630463685 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5f0fa7839174f51942b3285bec09c3ac in 1273ms, sequenceid=441, compaction requested=true at 1733630463689 (+4 ms)Writing region close event to WAL at 1733630463745 (+56 ms)Running coprocessor post-close hooks at 1733630463748 (+3 ms)Closed at 1733630463748 2024-12-08T04:01:03,750 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:01:03,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=5f0fa7839174f51942b3285bec09c3ac, regionState=CLOSED 2024-12-08T04:01:03,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:01:03,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-08T04:01:03,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f0fa7839174f51942b3285bec09c3ac, server=32df4b4bb439,39919,1733630308479 in 1.4900 sec 2024-12-08T04:01:03,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-08T04:01:03,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0fa7839174f51942b3285bec09c3ac, UNASSIGN in 1.4960 sec 2024-12-08T04:01:03,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-08T04:01:03,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5000 sec 2024-12-08T04:01:03,757 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630463756"}]},"ts":"1733630463756"} 2024-12-08T04:01:03,758 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T04:01:03,758 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T04:01:03,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5080 sec 2024-12-08T04:01:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T04:01:04,398 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:04,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T04:01:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,403 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T04:01:04,405 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,409 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:01:04,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/recovered.edits] 2024-12-08T04:01:04,416 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1ecfd28f38be41cc83646eaf18a83be9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/1ecfd28f38be41cc83646eaf18a83be9 2024-12-08T04:01:04,418 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d710525d60894d009bbc4ff59f44098d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d710525d60894d009bbc4ff59f44098d 2024-12-08T04:01:04,420 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/d8dcc4e2667c44d1b5dfa97a3d8ff0af 2024-12-08T04:01:04,422 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f3843cd649df4a8d8621b9fe771d2f56 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/A/f3843cd649df4a8d8621b9fe771d2f56 2024-12-08T04:01:04,426 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/2db0ef54daf8452491b11998f59ee685 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/2db0ef54daf8452491b11998f59ee685 2024-12-08T04:01:04,428 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/659ceb0cac8c4bbea65ae28a488df934 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/659ceb0cac8c4bbea65ae28a488df934 2024-12-08T04:01:04,431 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/92aabdb10cd64476aeb45a2d60954038 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/92aabdb10cd64476aeb45a2d60954038 2024-12-08T04:01:04,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/d11e028889f54eefaf08c5057b666497 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/B/d11e028889f54eefaf08c5057b666497 2024-12-08T04:01:04,437 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/6785d6934fc4488688a685039ef145ca to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/6785d6934fc4488688a685039ef145ca 2024-12-08T04:01:04,439 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b4c0288ceeab4002a45dd28ca455e9f1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/b4c0288ceeab4002a45dd28ca455e9f1 2024-12-08T04:01:04,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1164b1b62cf47859a22fe1f25d3f2ee to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/d1164b1b62cf47859a22fe1f25d3f2ee 2024-12-08T04:01:04,442 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/fd13da4e3faf46da95c43b7388a8845a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/C/fd13da4e3faf46da95c43b7388a8845a 2024-12-08T04:01:04,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/recovered.edits/444.seqid to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac/recovered.edits/444.seqid 2024-12-08T04:01:04,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/5f0fa7839174f51942b3285bec09c3ac 2024-12-08T04:01:04,446 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T04:01:04,449 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,451 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T04:01:04,453 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T04:01:04,454 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,454 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T04:01:04,454 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733630464454"}]},"ts":"9223372036854775807"} 2024-12-08T04:01:04,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T04:01:04,456 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5f0fa7839174f51942b3285bec09c3ac, NAME => 'TestAcidGuarantees,,1733630436752.5f0fa7839174f51942b3285bec09c3ac.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T04:01:04,456 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T04:01:04,457 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733630464456"}]},"ts":"9223372036854775807"} 2024-12-08T04:01:04,458 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T04:01:04,459 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-08T04:01:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-08T04:01:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T04:01:04,510 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:04,518 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=445 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 423), ProcessCount=11 (was 11), AvailableMemoryMB=16945 (was 16977) 2024-12-08T04:01:04,528 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=16944 2024-12-08T04:01:04,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T04:01:04,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:01:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:04,531 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T04:01:04,532 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:04,532 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-08T04:01:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T04:01:04,533 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T04:01:04,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742342_1518 (size=1061) 2024-12-08T04:01:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T04:01:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T04:01:04,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384 2024-12-08T04:01:04,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742343_1519 (size=53) 2024-12-08T04:01:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T04:01:05,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:05,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ca44a140da9733e9d0e45118f13fb6f0, disabling compactions & flushes 2024-12-08T04:01:05,353 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. after waiting 0 ms 2024-12-08T04:01:05,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,354 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,354 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca44a140da9733e9d0e45118f13fb6f0: Waiting for close lock at 1733630465353Disabling compacts and flushes for region at 1733630465353Disabling writes for close at 1733630465353Writing region close event to WAL at 1733630465353Closed at 1733630465353 2024-12-08T04:01:05,355 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T04:01:05,356 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733630465355"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630465355"}]},"ts":"1733630465355"} 2024-12-08T04:01:05,357 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T04:01:05,358 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T04:01:05,358 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630465358"}]},"ts":"1733630465358"} 2024-12-08T04:01:05,359 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T04:01:05,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, ASSIGN}] 2024-12-08T04:01:05,360 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, ASSIGN 2024-12-08T04:01:05,361 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, ASSIGN; state=OFFLINE, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=false 2024-12-08T04:01:05,512 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:05,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, ASSIGN because future has completed 2024-12-08T04:01:05,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:01:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T04:01:05,677 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,678 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:05,678 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,679 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:05,679 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,679 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,681 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,682 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:01:05,682 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca44a140da9733e9d0e45118f13fb6f0 columnFamilyName A 2024-12-08T04:01:05,682 DEBUG [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:05,683 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(327): Store=ca44a140da9733e9d0e45118f13fb6f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:05,683 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,684 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:01:05,684 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca44a140da9733e9d0e45118f13fb6f0 columnFamilyName B 2024-12-08T04:01:05,684 DEBUG [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:05,685 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(327): Store=ca44a140da9733e9d0e45118f13fb6f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:05,685 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,686 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:01:05,686 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca44a140da9733e9d0e45118f13fb6f0 columnFamilyName C 2024-12-08T04:01:05,686 DEBUG [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:05,686 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(327): Store=ca44a140da9733e9d0e45118f13fb6f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:05,687 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,687 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,688 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,688 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,689 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,689 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,690 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T04:01:05,691 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,693 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:05,693 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened ca44a140da9733e9d0e45118f13fb6f0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74181200, jitterRate=0.10538601875305176}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T04:01:05,693 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:05,694 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for ca44a140da9733e9d0e45118f13fb6f0: Running coprocessor pre-open hook at 1733630465679Writing region info on filesystem at 1733630465679Initializing all the Stores at 1733630465680 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630465680Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630465681 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630465681Cleaning up temporary data from old regions at 1733630465689 (+8 ms)Running coprocessor post-open hooks at 1733630465693 (+4 ms)Region opened successfully at 1733630465694 (+1 ms) 2024-12-08T04:01:05,695 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., pid=146, masterSystemTime=1733630465670 2024-12-08T04:01:05,697 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,697 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:05,698 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:05,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:01:05,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-08T04:01:05,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 in 184 msec 2024-12-08T04:01:05,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-08T04:01:05,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, ASSIGN in 343 msec 2024-12-08T04:01:05,705 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T04:01:05,706 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630465705"}]},"ts":"1733630465705"} 2024-12-08T04:01:05,707 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T04:01:05,708 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T04:01:05,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-08T04:01:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T04:01:06,679 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:06,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:06,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:06,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:06,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e52681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:06,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:06,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d886f96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:06,688 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:06,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:06,690 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T04:01:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,38065,1733630307764 2024-12-08T04:01:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b850df2 2024-12-08T04:01:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T04:01:06,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T04:01:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T04:01:06,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:01:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:06,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742344_1520 (size=1097) 2024-12-08T04:01:07,105 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T04:01:07,105 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T04:01:07,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T04:01:07,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, REOPEN/MOVE}] 2024-12-08T04:01:07,119 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, REOPEN/MOVE 2024-12-08T04:01:07,120 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, REOPEN/MOVE because future has completed 2024-12-08T04:01:07,121 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T04:01:07,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:01:07,274 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,274 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T04:01:07,275 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing ca44a140da9733e9d0e45118f13fb6f0, disabling compactions & flushes 2024-12-08T04:01:07,275 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,275 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,275 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. after waiting 0 ms 2024-12-08T04:01:07,275 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,281 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T04:01:07,282 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,282 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for ca44a140da9733e9d0e45118f13fb6f0: Waiting for close lock at 1733630467274Running coprocessor pre-close hooks at 1733630467275 (+1 ms)Disabling compacts and flushes for region at 1733630467275Disabling writes for close at 1733630467275Writing region close event to WAL at 1733630467276 (+1 ms)Running coprocessor post-close hooks at 1733630467281 (+5 ms)Closed at 1733630467281 2024-12-08T04:01:07,282 WARN [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: ca44a140da9733e9d0e45118f13fb6f0 to self. 2024-12-08T04:01:07,285 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,286 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=CLOSED 2024-12-08T04:01:07,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:01:07,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-08T04:01:07,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 in 167 msec 2024-12-08T04:01:07,291 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,39919,1733630308479; forceNewPlan=false, retain=true 2024-12-08T04:01:07,442 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=OPENING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, REOPEN/MOVE because future has completed 2024-12-08T04:01:07,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:01:07,601 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,601 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:07,601 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,601 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:07,602 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,602 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,603 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,603 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:01:07,604 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca44a140da9733e9d0e45118f13fb6f0 columnFamilyName A 2024-12-08T04:01:07,605 DEBUG [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:07,605 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(327): Store=ca44a140da9733e9d0e45118f13fb6f0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:07,605 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,606 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:01:07,606 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca44a140da9733e9d0e45118f13fb6f0 columnFamilyName B 2024-12-08T04:01:07,606 DEBUG [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:07,606 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(327): Store=ca44a140da9733e9d0e45118f13fb6f0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:07,607 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,607 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T04:01:07,607 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca44a140da9733e9d0e45118f13fb6f0 columnFamilyName C 2024-12-08T04:01:07,607 DEBUG [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:07,608 INFO [StoreOpener-ca44a140da9733e9d0e45118f13fb6f0-1 {}] regionserver.HStore(327): Store=ca44a140da9733e9d0e45118f13fb6f0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:07,608 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,608 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,609 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,610 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,611 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,611 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,611 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T04:01:07,613 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,614 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened ca44a140da9733e9d0e45118f13fb6f0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64757993, jitterRate=-0.035030707716941833}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T04:01:07,614 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,614 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for ca44a140da9733e9d0e45118f13fb6f0: Running coprocessor pre-open hook at 1733630467602Writing region info on filesystem at 1733630467602Initializing all the Stores at 1733630467602Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630467602Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630467603 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630467603Cleaning up temporary data from old regions at 1733630467611 (+8 ms)Running coprocessor post-open hooks at 1733630467614 (+3 ms)Region opened successfully at 1733630467614 2024-12-08T04:01:07,615 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., pid=151, masterSystemTime=1733630467598 2024-12-08T04:01:07,617 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,617 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,618 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=OPEN, openSeqNum=5, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:01:07,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-08T04:01:07,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 in 175 msec 2024-12-08T04:01:07,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-08T04:01:07,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, REOPEN/MOVE in 504 msec 2024-12-08T04:01:07,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-08T04:01:07,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-12-08T04:01:07,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-12-08T04:01:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-08T04:01:07,629 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:07,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d273c93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a238c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23965be2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749548ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,639 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87014c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675368cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,643 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491575f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3adfbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,647 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480d5982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,649 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5371fa54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2548f0fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,653 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@816c80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53195c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580a80a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,659 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe0292c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698d4e32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,663 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2f2b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,664 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636fa529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,38065,-1 for getting cluster id 2024-12-08T04:01:07,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:07,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9365ce4-250d-489c-a0fa-5ace31918096' 2024-12-08T04:01:07,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9365ce4-250d-489c-a0fa-5ace31918096" 2024-12-08T04:01:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cac0824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,38065,-1] 2024-12-08T04:01:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:07,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b8e2d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:07,669 DEBUG [Thread-2338 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,669 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,670 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,670 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,670 DEBUG [Thread-2327 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,670 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:07,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,671 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,672 DEBUG [Thread-2325 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,672 DEBUG [Thread-2329 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-08T04:01:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:07,673 DEBUG [Thread-2331 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,674 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,39919,1733630308479, seqNum=-1] 2024-12-08T04:01:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,676 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,679 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:07,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5] 2024-12-08T04:01:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:07,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:01:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:07,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630527718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630527720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630467714, max timestamp=1733630467720Num uniques 859; 2024-12-08T04:01:07,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:01:07,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630467714, max timestamp=1733630467720Num uniques 859; 2024-12-08T04:01:07,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630467714, max timestamp=1733630467720Num uniques 859; 2024-12-08T04:01:07,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:01:07,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:01:07,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208af33614376e1485aa18f442841d98d5f_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630467705/Put/seqid=0 2024-12-08T04:01:07,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742345_1521 (size=14594) 2024-12-08T04:01:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630527741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630527744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630527745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:07,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630527827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630527827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:07,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630527847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630527857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630527857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:07,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:07,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630528037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630528037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630528057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630528067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630528067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:08,137 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208af33614376e1485aa18f442841d98d5f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208af33614376e1485aa18f442841d98d5f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:08,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:08,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/f8d98b2790f641feb314258310a20cba, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/f8d98b2790f641feb314258310a20cba is 175, key is test_row_0/A:col10/1733630467705/Put/seqid=0 2024-12-08T04:01:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742346_1522 (size=39549) 2024-12-08T04:01:08,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630528347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630528348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630528367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630528377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630528377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,542 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/f8d98b2790f641feb314258310a20cba 2024-12-08T04:01:08,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/dd68a472f8da4894967a4c1d81b2cff9 is 50, key is test_row_0/B:col10/1733630467705/Put/seqid=0 2024-12-08T04:01:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742347_1523 (size=12001) 2024-12-08T04:01:08,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:08,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630528857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630528857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630528876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630528887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630528887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:08,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:08,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:08,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/dd68a472f8da4894967a4c1d81b2cff9 2024-12-08T04:01:08,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/0b5f468c058341f4829aeae561d20c41 is 50, key is test_row_0/C:col10/1733630467705/Put/seqid=0 2024-12-08T04:01:08,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742348_1524 (size=12001) 2024-12-08T04:01:09,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T04:01:09,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:09,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:09,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/0b5f468c058341f4829aeae561d20c41 2024-12-08T04:01:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/f8d98b2790f641feb314258310a20cba as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba 2024-12-08T04:01:09,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba, entries=200, sequenceid=15, filesize=38.6 K 2024-12-08T04:01:09,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/dd68a472f8da4894967a4c1d81b2cff9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/dd68a472f8da4894967a4c1d81b2cff9 2024-12-08T04:01:09,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/dd68a472f8da4894967a4c1d81b2cff9, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T04:01:09,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/0b5f468c058341f4829aeae561d20c41 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/0b5f468c058341f4829aeae561d20c41 2024-12-08T04:01:09,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/0b5f468c058341f4829aeae561d20c41, entries=150, sequenceid=15, filesize=11.7 K 2024-12-08T04:01:09,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ca44a140da9733e9d0e45118f13fb6f0 in 1693ms, sequenceid=15, compaction requested=false 2024-12-08T04:01:09,398 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T04:01:09,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:09,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T04:01:09,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:09,510 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-08T04:01:09,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:09,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:09,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a6f8a24c420a4d77811f303e783a8a4f_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630467739/Put/seqid=0 2024-12-08T04:01:09,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742349_1525 (size=12154) 2024-12-08T04:01:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:09,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630529886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630529887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:09,890 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630529889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630529897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:09,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630529897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:09,900 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:09,900 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:09,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:09,921 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a6f8a24c420a4d77811f303e783a8a4f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a6f8a24c420a4d77811f303e783a8a4f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/80cda37ca2a8473eae13aa0131e9657e, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/80cda37ca2a8473eae13aa0131e9657e is 175, key is test_row_0/A:col10/1733630467739/Put/seqid=0 2024-12-08T04:01:09,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742350_1526 (size=30955) 2024-12-08T04:01:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630529997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630529997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630530207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:10,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630530207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:10,326 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/80cda37ca2a8473eae13aa0131e9657e 2024-12-08T04:01:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/c482e20a993246ff9db451b5dedd6fce is 50, key is test_row_0/B:col10/1733630467739/Put/seqid=0 2024-12-08T04:01:10,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742351_1527 (size=12001) 2024-12-08T04:01:10,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630530517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630530517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:10,734 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/c482e20a993246ff9db451b5dedd6fce 2024-12-08T04:01:10,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/b6daee9c378649588df5de2b21c4fe00 is 50, key is test_row_0/C:col10/1733630467739/Put/seqid=0 2024-12-08T04:01:10,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742352_1528 (size=12001) 2024-12-08T04:01:10,742 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/b6daee9c378649588df5de2b21c4fe00 2024-12-08T04:01:10,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/80cda37ca2a8473eae13aa0131e9657e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e 2024-12-08T04:01:10,748 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e, entries=150, sequenceid=50, filesize=30.2 K 2024-12-08T04:01:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/c482e20a993246ff9db451b5dedd6fce as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/c482e20a993246ff9db451b5dedd6fce 2024-12-08T04:01:10,753 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/c482e20a993246ff9db451b5dedd6fce, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T04:01:10,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/b6daee9c378649588df5de2b21c4fe00 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/b6daee9c378649588df5de2b21c4fe00 2024-12-08T04:01:10,756 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/b6daee9c378649588df5de2b21c4fe00, entries=150, sequenceid=50, filesize=11.7 K 2024-12-08T04:01:10,757 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ca44a140da9733e9d0e45118f13fb6f0 in 1247ms, sequenceid=50, compaction requested=false 2024-12-08T04:01:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-08T04:01:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-08T04:01:10,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-08T04:01:10,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0790 sec 2024-12-08T04:01:10,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0910 sec 2024-12-08T04:01:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:11,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T04:01:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:11,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082056da7cfc434e43b0b21bb502f63a2c_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:11,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742353_1529 (size=14594) 2024-12-08T04:01:11,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630531063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630531064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630531177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630531177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630531387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630531387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,441 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:11,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082056da7cfc434e43b0b21bb502f63a2c_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082056da7cfc434e43b0b21bb502f63a2c_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:11,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/6a0d1a4537054aedaecb3de95a06a5b1, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:11,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/6a0d1a4537054aedaecb3de95a06a5b1 is 175, key is test_row_0/A:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:11,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742354_1530 (size=39549) 2024-12-08T04:01:11,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/6a0d1a4537054aedaecb3de95a06a5b1 2024-12-08T04:01:11,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/a8b8330231764036862dda209937aa8b is 50, key is test_row_0/B:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:11,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742355_1531 (size=12001) 2024-12-08T04:01:11,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630531697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630531697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T04:01:11,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:11,837 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-08T04:01:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T04:01:11,839 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:11,839 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:11,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:11,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/a8b8330231764036862dda209937aa8b 2024-12-08T04:01:11,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/6c5cef8524ee409790362e1334bf8829 is 50, key is test_row_0/C:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742356_1532 (size=12001) 2024-12-08T04:01:11,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630531907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,910 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630531917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630531917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:11,919 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:11,920 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T04:01:11,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T04:01:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:11,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:12,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T04:01:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T04:01:12,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630532207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:12,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630532207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:12,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/6c5cef8524ee409790362e1334bf8829 2024-12-08T04:01:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/6a0d1a4537054aedaecb3de95a06a5b1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1 2024-12-08T04:01:12,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1, entries=200, sequenceid=63, filesize=38.6 K 2024-12-08T04:01:12,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/a8b8330231764036862dda209937aa8b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/a8b8330231764036862dda209937aa8b 2024-12-08T04:01:12,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/a8b8330231764036862dda209937aa8b, entries=150, sequenceid=63, filesize=11.7 K 2024-12-08T04:01:12,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/6c5cef8524ee409790362e1334bf8829 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/6c5cef8524ee409790362e1334bf8829 2024-12-08T04:01:12,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/6c5cef8524ee409790362e1334bf8829, entries=150, sequenceid=63, filesize=11.7 K 2024-12-08T04:01:12,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ca44a140da9733e9d0e45118f13fb6f0 in 1248ms, sequenceid=63, compaction requested=true 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:12,278 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:12,278 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:12,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:12,278 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:12,278 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:12,278 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:12,278 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:12,278 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,278 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,278 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=107.5 K 2024-12-08T04:01:12,278 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/dd68a472f8da4894967a4c1d81b2cff9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/c482e20a993246ff9db451b5dedd6fce, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/a8b8330231764036862dda209937aa8b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=35.2 K 2024-12-08T04:01:12,279 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1] 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting dd68a472f8da4894967a4c1d81b2cff9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733630467678 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d98b2790f641feb314258310a20cba, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733630467678 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c482e20a993246ff9db451b5dedd6fce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733630467739 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80cda37ca2a8473eae13aa0131e9657e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733630467739 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b8330231764036862dda209937aa8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733630469876 2024-12-08T04:01:12,279 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0d1a4537054aedaecb3de95a06a5b1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733630469876 2024-12-08T04:01:12,284 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:12,284 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:12,284 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f62e77cdf81d4ad1b233fbfb06d32ad3 is 50, key is test_row_0/B:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:12,286 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208b5dad64efecf4a6f92866e0dca0a8f54_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:12,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742357_1533 (size=12104) 2024-12-08T04:01:12,287 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208b5dad64efecf4a6f92866e0dca0a8f54_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:12,287 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b5dad64efecf4a6f92866e0dca0a8f54_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:12,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742358_1534 (size=4469) 2024-12-08T04:01:12,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T04:01:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,296 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T04:01:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:12,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:12,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ec9c2d3c805449679d9f6fff62d700ad_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630471057/Put/seqid=0 2024-12-08T04:01:12,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742359_1535 (size=12154) 2024-12-08T04:01:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T04:01:12,690 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f62e77cdf81d4ad1b233fbfb06d32ad3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f62e77cdf81d4ad1b233fbfb06d32ad3 2024-12-08T04:01:12,691 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#456 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:12,696 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/5d6fa5d4fd194f9889f29101d259596e is 175, key is test_row_0/A:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:12,699 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into f62e77cdf81d4ad1b233fbfb06d32ad3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:12,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742360_1536 (size=31058) 2024-12-08T04:01:12,699 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:12,699 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=13, startTime=1733630472278; duration=0sec 2024-12-08T04:01:12,699 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:12,699 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:12,699 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:12,700 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:12,700 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:12,700 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:12,700 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/0b5f468c058341f4829aeae561d20c41, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/b6daee9c378649588df5de2b21c4fe00, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/6c5cef8524ee409790362e1334bf8829] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=35.2 K 2024-12-08T04:01:12,700 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5f468c058341f4829aeae561d20c41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733630467678 2024-12-08T04:01:12,701 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting b6daee9c378649588df5de2b21c4fe00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733630467739 2024-12-08T04:01:12,701 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5cef8524ee409790362e1334bf8829, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733630469876 2024-12-08T04:01:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:12,705 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:12,706 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/d187185e34f44510ba894f98541084b4 is 50, key is test_row_0/C:col10/1733630469876/Put/seqid=0 2024-12-08T04:01:12,707 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ec9c2d3c805449679d9f6fff62d700ad_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ec9c2d3c805449679d9f6fff62d700ad_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:12,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/896f4f890c7f47e7986d5ecf4161f386, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:12,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/896f4f890c7f47e7986d5ecf4161f386 is 175, key is test_row_0/A:col10/1733630471057/Put/seqid=0 2024-12-08T04:01:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742362_1538 (size=30955) 2024-12-08T04:01:12,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742361_1537 (size=12104) 2024-12-08T04:01:12,722 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/d187185e34f44510ba894f98541084b4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d187185e34f44510ba894f98541084b4 2024-12-08T04:01:12,726 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into d187185e34f44510ba894f98541084b4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:12,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:12,726 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=13, startTime=1733630472278; duration=0sec 2024-12-08T04:01:12,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:12,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T04:01:13,103 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/5d6fa5d4fd194f9889f29101d259596e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/5d6fa5d4fd194f9889f29101d259596e 2024-12-08T04:01:13,107 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into 5d6fa5d4fd194f9889f29101d259596e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:13,107 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:13,108 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=13, startTime=1733630472278; duration=0sec 2024-12-08T04:01:13,108 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:13,108 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:13,113 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/896f4f890c7f47e7986d5ecf4161f386 2024-12-08T04:01:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/895459cfde264e2abc5ff790125cb876 is 50, key is test_row_0/B:col10/1733630471057/Put/seqid=0 2024-12-08T04:01:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742363_1539 (size=12001) 2024-12-08T04:01:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:13,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630533235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630533238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630533346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630533347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,521 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/895459cfde264e2abc5ff790125cb876 2024-12-08T04:01:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/a1279ad0389943128f47c6db5f2118ef is 50, key is test_row_0/C:col10/1733630471057/Put/seqid=0 2024-12-08T04:01:13,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742364_1540 (size=12001) 2024-12-08T04:01:13,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630533557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630533557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630533867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630533867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:13,930 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/a1279ad0389943128f47c6db5f2118ef 2024-12-08T04:01:13,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/896f4f890c7f47e7986d5ecf4161f386 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386 2024-12-08T04:01:13,936 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386, entries=150, sequenceid=82, filesize=30.2 K 2024-12-08T04:01:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/895459cfde264e2abc5ff790125cb876 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/895459cfde264e2abc5ff790125cb876 2024-12-08T04:01:13,939 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/895459cfde264e2abc5ff790125cb876, entries=150, sequenceid=82, filesize=11.7 K 2024-12-08T04:01:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/a1279ad0389943128f47c6db5f2118ef as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a1279ad0389943128f47c6db5f2118ef 2024-12-08T04:01:13,943 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a1279ad0389943128f47c6db5f2118ef, entries=150, sequenceid=82, filesize=11.7 K 2024-12-08T04:01:13,943 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ca44a140da9733e9d0e45118f13fb6f0 in 1647ms, sequenceid=82, compaction requested=false 2024-12-08T04:01:13,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:13,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:13,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-08T04:01:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-08T04:01:13,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-08T04:01:13,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1060 sec 2024-12-08T04:01:13,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1090 sec 2024-12-08T04:01:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T04:01:13,987 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:13,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-08T04:01:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T04:01:13,989 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:13,990 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:13,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T04:01:14,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T04:01:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:14,142 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T04:01:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:14,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:14,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080015ef19b7e24b8badeb372ac037bcdb_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630473229/Put/seqid=0 2024-12-08T04:01:14,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742365_1541 (size=12154) 2024-12-08T04:01:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:14,153 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080015ef19b7e24b8badeb372ac037bcdb_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080015ef19b7e24b8badeb372ac037bcdb_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/59c4b2b9b8ab4248a9e5b632a042f1e8, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/59c4b2b9b8ab4248a9e5b632a042f1e8 is 175, key is test_row_0/A:col10/1733630473229/Put/seqid=0 2024-12-08T04:01:14,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742366_1542 (size=30955) 2024-12-08T04:01:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T04:01:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:14,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:14,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630534406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630534407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630534517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630534517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:14,557 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/59c4b2b9b8ab4248a9e5b632a042f1e8 2024-12-08T04:01:14,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/ac1f7c518b0b49f5a228e5982b0e4cd0 is 50, key is test_row_0/B:col10/1733630473229/Put/seqid=0 2024-12-08T04:01:14,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742367_1543 (size=12001) 2024-12-08T04:01:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T04:01:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630534727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:14,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630534727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:14,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:14,966 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/ac1f7c518b0b49f5a228e5982b0e4cd0 2024-12-08T04:01:14,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/a86ddc3a781141559e2ce0c0ebbc6e9c is 50, key is test_row_0/C:col10/1733630473229/Put/seqid=0 2024-12-08T04:01:14,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742368_1544 (size=12001) 2024-12-08T04:01:14,985 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/a86ddc3a781141559e2ce0c0ebbc6e9c 2024-12-08T04:01:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/59c4b2b9b8ab4248a9e5b632a042f1e8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8 2024-12-08T04:01:14,993 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8, entries=150, sequenceid=98, filesize=30.2 K 2024-12-08T04:01:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/ac1f7c518b0b49f5a228e5982b0e4cd0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ac1f7c518b0b49f5a228e5982b0e4cd0 2024-12-08T04:01:14,996 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ac1f7c518b0b49f5a228e5982b0e4cd0, entries=150, sequenceid=98, filesize=11.7 K 2024-12-08T04:01:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/a86ddc3a781141559e2ce0c0ebbc6e9c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a86ddc3a781141559e2ce0c0ebbc6e9c 2024-12-08T04:01:15,000 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a86ddc3a781141559e2ce0c0ebbc6e9c, entries=150, sequenceid=98, filesize=11.7 K 2024-12-08T04:01:15,001 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ca44a140da9733e9d0e45118f13fb6f0 in 859ms, sequenceid=98, compaction requested=true 2024-12-08T04:01:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-08T04:01:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-08T04:01:15,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-08T04:01:15,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-12-08T04:01:15,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0180 sec 2024-12-08T04:01:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:15,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T04:01:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:15,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:15,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208be90d076c2f24c1fbf4ce412cc7db006_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:15,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742369_1545 (size=12154) 2024-12-08T04:01:15,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630535061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630535061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T04:01:15,127 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:15,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-08T04:01:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T04:01:15,129 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:15,130 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:15,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630535167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630535177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T04:01:15,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T04:01:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:15,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630535377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630535387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T04:01:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T04:01:15,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208be90d076c2f24c1fbf4ce412cc7db006_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208be90d076c2f24c1fbf4ce412cc7db006_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:15,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/a355fafcb02740dd9f9cba84d25c91fc, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:15,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/a355fafcb02740dd9f9cba84d25c91fc is 175, key is test_row_0/A:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:15,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742370_1546 (size=30955) 2024-12-08T04:01:15,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T04:01:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630535687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630535697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:15,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:15,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T04:01:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T04:01:15,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/a355fafcb02740dd9f9cba84d25c91fc 2024-12-08T04:01:15,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/d5e9efc2078c4c64861b854e491c2c3b is 50, key is test_row_0/B:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:15,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742371_1547 (size=12001) 2024-12-08T04:01:15,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/d5e9efc2078c4c64861b854e491c2c3b 2024-12-08T04:01:15,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8792e3803e23436b84e991c3d6518c91 is 50, key is test_row_0/C:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:15,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742372_1548 (size=12001) 2024-12-08T04:01:15,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8792e3803e23436b84e991c3d6518c91 2024-12-08T04:01:15,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/a355fafcb02740dd9f9cba84d25c91fc as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc 2024-12-08T04:01:15,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc, entries=150, sequenceid=119, filesize=30.2 K 2024-12-08T04:01:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/d5e9efc2078c4c64861b854e491c2c3b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d5e9efc2078c4c64861b854e491c2c3b 2024-12-08T04:01:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d5e9efc2078c4c64861b854e491c2c3b, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T04:01:15,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8792e3803e23436b84e991c3d6518c91 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8792e3803e23436b84e991c3d6518c91 2024-12-08T04:01:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8792e3803e23436b84e991c3d6518c91, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T04:01:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ca44a140da9733e9d0e45118f13fb6f0 in 850ms, sequenceid=119, compaction requested=true 2024-12-08T04:01:15,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:15,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:15,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:15,890 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:01:15,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,890 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:01:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:15,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:15,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:15,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T04:01:15,892 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:01:15,892 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,892 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,892 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T04:01:15,892 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/5d6fa5d4fd194f9889f29101d259596e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=121.0 K 2024-12-08T04:01:15,892 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,892 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/5d6fa5d4fd194f9889f29101d259596e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc] 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:15,894 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d6fa5d4fd194f9889f29101d259596e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733630469876 2024-12-08T04:01:15,894 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:01:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,894 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:15,894 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:15,894 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f62e77cdf81d4ad1b233fbfb06d32ad3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/895459cfde264e2abc5ff790125cb876, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ac1f7c518b0b49f5a228e5982b0e4cd0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d5e9efc2078c4c64861b854e491c2c3b] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=47.0 K 2024-12-08T04:01:15,894 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 896f4f890c7f47e7986d5ecf4161f386, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733630471046 2024-12-08T04:01:15,895 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f62e77cdf81d4ad1b233fbfb06d32ad3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733630469876 2024-12-08T04:01:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,895 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c4b2b9b8ab4248a9e5b632a042f1e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733630473229 2024-12-08T04:01:15,895 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 895459cfde264e2abc5ff790125cb876, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733630471046 2024-12-08T04:01:15,895 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting a355fafcb02740dd9f9cba84d25c91fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733630474401 2024-12-08T04:01:15,895 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ac1f7c518b0b49f5a228e5982b0e4cd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733630473229 2024-12-08T04:01:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,895 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d5e9efc2078c4c64861b854e491c2c3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733630474401 2024-12-08T04:01:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e1c596c6b0924f30bce7378674547839_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630475053/Put/seqid=0 2024-12-08T04:01:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742373_1549 (size=9764) 2024-12-08T04:01:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,908 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:15,908 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:15,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,909 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/75265b7e7a76441a9899e9716356cd69 is 50, key is test_row_0/B:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,913 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e1c596c6b0924f30bce7378674547839_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e1c596c6b0924f30bce7378674547839_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/9cd05e7292504a9b83d187b41b0e1a41, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/9cd05e7292504a9b83d187b41b0e1a41 is 175, key is test_row_0/A:col10/1733630475053/Put/seqid=0 2024-12-08T04:01:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,919 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120813c2ee0ec2944db8a07cfab41cf5837d_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,922 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120813c2ee0ec2944db8a07cfab41cf5837d_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,922 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120813c2ee0ec2944db8a07cfab41cf5837d_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742374_1550 (size=12241) 2024-12-08T04:01:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742375_1551 (size=22411) 2024-12-08T04:01:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742376_1552 (size=4469) 2024-12-08T04:01:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:15,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630475964, max timestamp=1733630475972Num uniques 859; 2024-12-08T04:01:15,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T04:01:15,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630475964, max timestamp=1733630475972Num uniques 859; 2024-12-08T04:01:15,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T04:01:15,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733630475964, max timestamp=1733630475972Num uniques 859; 2024-12-08T04:01:15,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T04:01:15,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630535994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630535999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630535999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630536107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630536117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630536117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630536197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630536207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T04:01:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630536317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,328 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/75265b7e7a76441a9899e9716356cd69 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/75265b7e7a76441a9899e9716356cd69 2024-12-08T04:01:16,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630536327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630536327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,331 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into 75265b7e7a76441a9899e9716356cd69(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:16,331 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:16,331 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=12, startTime=1733630475890; duration=0sec 2024-12-08T04:01:16,331 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:16,331 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:16,331 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T04:01:16,332 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T04:01:16,332 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:16,332 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:16,332 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d187185e34f44510ba894f98541084b4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a1279ad0389943128f47c6db5f2118ef, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a86ddc3a781141559e2ce0c0ebbc6e9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8792e3803e23436b84e991c3d6518c91] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=47.0 K 2024-12-08T04:01:16,333 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d187185e34f44510ba894f98541084b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733630469876 2024-12-08T04:01:16,333 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a1279ad0389943128f47c6db5f2118ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733630471046 2024-12-08T04:01:16,333 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting a86ddc3a781141559e2ce0c0ebbc6e9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733630473229 2024-12-08T04:01:16,333 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8792e3803e23436b84e991c3d6518c91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733630474401 2024-12-08T04:01:16,336 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/9cd05e7292504a9b83d187b41b0e1a41 2024-12-08T04:01:16,337 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:16,338 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/0c8a8eb89ea14fa086b60e4ba90336f6 is 175, key is test_row_0/A:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:16,343 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:16,344 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/5f7cd811e3614fd9a328c91c67405613 is 50, key is test_row_0/C:col10/1733630474401/Put/seqid=0 2024-12-08T04:01:16,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742377_1553 (size=31195) 2024-12-08T04:01:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/d89b376c6e624135b50ab8a59d666f31 is 50, key is test_row_0/B:col10/1733630475053/Put/seqid=0 2024-12-08T04:01:16,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742378_1554 (size=12241) 2024-12-08T04:01:16,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742379_1555 (size=9707) 2024-12-08T04:01:16,365 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/d89b376c6e624135b50ab8a59d666f31 2024-12-08T04:01:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/4b3d704038184f82a6aca7e3ed9f6dc1 is 50, key is test_row_0/C:col10/1733630475053/Put/seqid=0 2024-12-08T04:01:16,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742380_1556 (size=9707) 2024-12-08T04:01:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630536627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630536637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630536637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:16,752 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/0c8a8eb89ea14fa086b60e4ba90336f6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0c8a8eb89ea14fa086b60e4ba90336f6 2024-12-08T04:01:16,756 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into 0c8a8eb89ea14fa086b60e4ba90336f6(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:16,756 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:16,756 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=12, startTime=1733630475890; duration=0sec 2024-12-08T04:01:16,756 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:16,756 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:16,763 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/5f7cd811e3614fd9a328c91c67405613 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/5f7cd811e3614fd9a328c91c67405613 2024-12-08T04:01:16,766 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into 5f7cd811e3614fd9a328c91c67405613(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:16,766 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:16,766 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=12, startTime=1733630475891; duration=0sec 2024-12-08T04:01:16,767 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:16,767 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:16,773 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/4b3d704038184f82a6aca7e3ed9f6dc1 2024-12-08T04:01:16,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/9cd05e7292504a9b83d187b41b0e1a41 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41 2024-12-08T04:01:16,779 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41, entries=100, sequenceid=130, filesize=21.9 K 2024-12-08T04:01:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/d89b376c6e624135b50ab8a59d666f31 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d89b376c6e624135b50ab8a59d666f31 2024-12-08T04:01:16,787 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d89b376c6e624135b50ab8a59d666f31, entries=100, sequenceid=130, filesize=9.5 K 2024-12-08T04:01:16,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/4b3d704038184f82a6aca7e3ed9f6dc1 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4b3d704038184f82a6aca7e3ed9f6dc1 2024-12-08T04:01:16,791 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4b3d704038184f82a6aca7e3ed9f6dc1, entries=100, sequenceid=130, filesize=9.5 K 2024-12-08T04:01:16,792 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ca44a140da9733e9d0e45118f13fb6f0 in 900ms, sequenceid=130, compaction requested=false 2024-12-08T04:01:16,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:16,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:16,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-08T04:01:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-08T04:01:16,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-08T04:01:16,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6630 sec 2024-12-08T04:01:16,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6670 sec 2024-12-08T04:01:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:17,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T04:01:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:17,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208280a247fa3cc4e98bb30e462ff896128_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:17,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742381_1557 (size=14794) 2024-12-08T04:01:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630537154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630537154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630537155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630537207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,210 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630537216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,219 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630537267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630537267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630537267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T04:01:17,278 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:17,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-08T04:01:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:17,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:17,280 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:17,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:17,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630537477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630537477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630537477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:17,552 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208280a247fa3cc4e98bb30e462ff896128_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208280a247fa3cc4e98bb30e462ff896128_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:17,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/401c1f1653bc438da17a6bedaa508709, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:17,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/401c1f1653bc438da17a6bedaa508709 is 175, key is test_row_0/A:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:17,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742382_1558 (size=39749) 2024-12-08T04:01:17,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:17,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630537787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630537787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630537787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:17,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:17,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:17,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/401c1f1653bc438da17a6bedaa508709 2024-12-08T04:01:17,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/cd591f46d6b6441ab8d3c8040fc6def5 is 50, key is test_row_0/B:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:17,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742383_1559 (size=12151) 2024-12-08T04:01:18,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:18,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:18,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630538297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:18,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630538297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:18,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630538297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:18,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:18,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:18,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/cd591f46d6b6441ab8d3c8040fc6def5 2024-12-08T04:01:18,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/cb0cf26c11154bd1b63cedf62bf4fab3 is 50, key is test_row_0/C:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:18,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742384_1560 (size=12151) 2024-12-08T04:01:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:18,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:18,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:18,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:18,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/cb0cf26c11154bd1b63cedf62bf4fab3 2024-12-08T04:01:18,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/401c1f1653bc438da17a6bedaa508709 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709 2024-12-08T04:01:18,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709, entries=200, sequenceid=168, filesize=38.8 K 2024-12-08T04:01:18,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/cd591f46d6b6441ab8d3c8040fc6def5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/cd591f46d6b6441ab8d3c8040fc6def5 2024-12-08T04:01:18,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/cd591f46d6b6441ab8d3c8040fc6def5, entries=150, sequenceid=168, filesize=11.9 K 2024-12-08T04:01:18,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/cb0cf26c11154bd1b63cedf62bf4fab3 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/cb0cf26c11154bd1b63cedf62bf4fab3 2024-12-08T04:01:18,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/cb0cf26c11154bd1b63cedf62bf4fab3, entries=150, sequenceid=168, filesize=11.9 K 2024-12-08T04:01:18,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ca44a140da9733e9d0e45118f13fb6f0 in 1648ms, sequenceid=168, compaction requested=true 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:18,788 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:18,788 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:18,795 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:18,795 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:18,795 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:18,795 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:18,795 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,795 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,796 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0c8a8eb89ea14fa086b60e4ba90336f6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=91.2 K 2024-12-08T04:01:18,796 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/75265b7e7a76441a9899e9716356cd69, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d89b376c6e624135b50ab8a59d666f31, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/cd591f46d6b6441ab8d3c8040fc6def5] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=33.3 K 2024-12-08T04:01:18,796 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,796 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0c8a8eb89ea14fa086b60e4ba90336f6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709] 2024-12-08T04:01:18,796 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 75265b7e7a76441a9899e9716356cd69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733630474401 2024-12-08T04:01:18,796 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8a8eb89ea14fa086b60e4ba90336f6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733630474401 2024-12-08T04:01:18,796 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d89b376c6e624135b50ab8a59d666f31, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733630475053 2024-12-08T04:01:18,796 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cd05e7292504a9b83d187b41b0e1a41, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733630475053 2024-12-08T04:01:18,796 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting cd591f46d6b6441ab8d3c8040fc6def5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733630475977 2024-12-08T04:01:18,797 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 401c1f1653bc438da17a6bedaa508709, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733630475977 2024-12-08T04:01:18,801 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:18,802 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:18,803 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/2a20f7b260e44995a865a77ecb794599 is 50, key is test_row_0/B:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:18,803 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412083d045755bc7b4b4a930aa55a41b9c6fa_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:18,804 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412083d045755bc7b4b4a930aa55a41b9c6fa_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:18,804 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083d045755bc7b4b4a930aa55a41b9c6fa_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:18,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T04:01:18,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,812 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T04:01:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:18,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:18,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742386_1562 (size=4469) 2024-12-08T04:01:18,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742385_1561 (size=12493) 2024-12-08T04:01:18,835 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#477 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:18,836 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/453cfb5cdc8347dd86427552be555680 is 175, key is test_row_0/A:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:18,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d475a53119d04e98a429e80caadf98e4_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630477153/Put/seqid=0 2024-12-08T04:01:18,840 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/2a20f7b260e44995a865a77ecb794599 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/2a20f7b260e44995a865a77ecb794599 2024-12-08T04:01:18,843 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into 2a20f7b260e44995a865a77ecb794599(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:18,843 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:18,843 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=13, startTime=1733630478788; duration=0sec 2024-12-08T04:01:18,843 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:18,843 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:18,843 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:18,844 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:18,844 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:18,844 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:18,845 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/5f7cd811e3614fd9a328c91c67405613, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4b3d704038184f82a6aca7e3ed9f6dc1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/cb0cf26c11154bd1b63cedf62bf4fab3] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=33.3 K 2024-12-08T04:01:18,847 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f7cd811e3614fd9a328c91c67405613, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733630474401 2024-12-08T04:01:18,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742387_1563 (size=31447) 2024-12-08T04:01:18,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742388_1564 (size=12304) 2024-12-08T04:01:18,847 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3d704038184f82a6aca7e3ed9f6dc1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733630475053 2024-12-08T04:01:18,847 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0cf26c11154bd1b63cedf62bf4fab3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733630475977 2024-12-08T04:01:18,852 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:18,852 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/c1bee2df55b74af6af02cb745e4c5e32 is 50, key is test_row_0/C:col10/1733630475993/Put/seqid=0 2024-12-08T04:01:18,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742389_1565 (size=12493) 2024-12-08T04:01:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:19,250 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d475a53119d04e98a429e80caadf98e4_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d475a53119d04e98a429e80caadf98e4_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:19,251 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/453cfb5cdc8347dd86427552be555680 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/453cfb5cdc8347dd86427552be555680 2024-12-08T04:01:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/fd70845e4dab4173b08eee55c1187440, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/fd70845e4dab4173b08eee55c1187440 is 175, key is test_row_0/A:col10/1733630477153/Put/seqid=0 2024-12-08T04:01:19,256 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into 453cfb5cdc8347dd86427552be555680(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:19,256 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:19,256 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=13, startTime=1733630478788; duration=0sec 2024-12-08T04:01:19,256 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:19,256 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:19,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742390_1566 (size=31105) 2024-12-08T04:01:19,266 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/c1bee2df55b74af6af02cb745e4c5e32 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/c1bee2df55b74af6af02cb745e4c5e32 2024-12-08T04:01:19,272 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into c1bee2df55b74af6af02cb745e4c5e32(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:19,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:19,272 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=13, startTime=1733630478788; duration=0sec 2024-12-08T04:01:19,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:19,272 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:19,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630479264, max timestamp=1733630479277Num uniques 859; 2024-12-08T04:01:19,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:01:19,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630479264, max timestamp=1733630479277Num uniques 859; 2024-12-08T04:01:19,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733630479264, max timestamp=1733630479277Num uniques 859; 2024-12-08T04:01:19,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:01:19,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T04:01:19,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733630479301, max timestamp=1733630479310Num uniques 1018; 2024-12-08T04:01:19,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733630479301, max timestamp=1733630479310Num uniques 1018; 2024-12-08T04:01:19,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T04:01:19,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733630479301, max timestamp=1733630479310Num uniques 1018; 2024-12-08T04:01:19,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T04:01:19,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T04:01:19,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630539327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630539330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630539331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630539331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630539332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630539437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630539437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630539447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630539447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630539447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630539647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630539647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630539657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630539657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630539657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,661 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/fd70845e4dab4173b08eee55c1187440 2024-12-08T04:01:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/e0541c4b7cb04bb484ad44c24cd34876 is 50, key is test_row_0/B:col10/1733630477153/Put/seqid=0 2024-12-08T04:01:19,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742391_1567 (size=12151) 2024-12-08T04:01:19,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630539957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630539957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630539967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630539967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630539967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:19,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:20,070 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/e0541c4b7cb04bb484ad44c24cd34876 2024-12-08T04:01:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/51c71f095e334dc294c5c5ec8fe188e6 is 50, key is test_row_0/C:col10/1733630477153/Put/seqid=0 2024-12-08T04:01:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742392_1568 (size=12151) 2024-12-08T04:01:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630540467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630540468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:20,478 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/51c71f095e334dc294c5c5ec8fe188e6 2024-12-08T04:01:20,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630540477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630540477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630540477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:20,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/fd70845e4dab4173b08eee55c1187440 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440 2024-12-08T04:01:20,484 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440, entries=150, sequenceid=178, filesize=30.4 K 2024-12-08T04:01:20,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/e0541c4b7cb04bb484ad44c24cd34876 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/e0541c4b7cb04bb484ad44c24cd34876 2024-12-08T04:01:20,487 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/e0541c4b7cb04bb484ad44c24cd34876, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T04:01:20,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/51c71f095e334dc294c5c5ec8fe188e6 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/51c71f095e334dc294c5c5ec8fe188e6 2024-12-08T04:01:20,490 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/51c71f095e334dc294c5c5ec8fe188e6, entries=150, sequenceid=178, filesize=11.9 K 2024-12-08T04:01:20,490 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ca44a140da9733e9d0e45118f13fb6f0 in 1678ms, sequenceid=178, compaction requested=false 2024-12-08T04:01:20,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:20,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:20,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-08T04:01:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-08T04:01:20,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-08T04:01:20,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2230 sec 2024-12-08T04:01:20,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.2280 sec 2024-12-08T04:01:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T04:01:21,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:21,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-08T04:01:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:21,449 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:21,449 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:21,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:21,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=140.89 KB heapSize=399.61 KB 2024-12-08T04:01:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T04:01:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T04:01:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T04:01:21,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120867b1f076649f480198a684449993ef7f_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630541496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,500 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_2' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630541497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630541498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,501 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_0' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630541498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742393_1569 (size=17284) 2024-12-08T04:01:21,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630541500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:21,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:21,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630541607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630541607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630541607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:21,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:21,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630541816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630541817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630541817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:21,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:21,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120867b1f076649f480198a684449993ef7f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120867b1f076649f480198a684449993ef7f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:21,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:21,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/29cffe3c0452468489c27f08b23448a8, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:21,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/29cffe3c0452468489c27f08b23448a8 is 175, key is test_row_0/A:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:21,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742394_1570 (size=48389) 2024-12-08T04:01:22,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:22,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630542127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630542127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630542127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:22,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/29cffe3c0452468489c27f08b23448a8 2024-12-08T04:01:22,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/de54995b62854e3c80d7ababc15b25fc is 50, key is test_row_0/B:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:22,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742395_1571 (size=12151) 2024-12-08T04:01:22,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630542637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630542637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630542637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:22,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/de54995b62854e3c80d7ababc15b25fc 2024-12-08T04:01:22,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/94db48d553ec497b862e938753b736bb is 50, key is test_row_0/C:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742396_1572 (size=12151) 2024-12-08T04:01:22,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:22,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:23,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:23,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:23,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/94db48d553ec497b862e938753b736bb 2024-12-08T04:01:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/29cffe3c0452468489c27f08b23448a8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8 2024-12-08T04:01:23,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8, entries=250, sequenceid=237, filesize=47.3 K 2024-12-08T04:01:23,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/de54995b62854e3c80d7ababc15b25fc as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/de54995b62854e3c80d7ababc15b25fc 2024-12-08T04:01:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/de54995b62854e3c80d7ababc15b25fc, entries=150, sequenceid=237, filesize=11.9 K 2024-12-08T04:01:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/94db48d553ec497b862e938753b736bb as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/94db48d553ec497b862e938753b736bb 2024-12-08T04:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/94db48d553ec497b862e938753b736bb, entries=150, sequenceid=237, filesize=11.9 K 2024-12-08T04:01:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~439.88 KB/450432, currentSize=26.84 KB/27480 for ca44a140da9733e9d0e45118f13fb6f0 in 1651ms, sequenceid=237, compaction requested=true 2024-12-08T04:01:23,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:23,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:23,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:23,141 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:23,141 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:23,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:23,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:23,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:23,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,142 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:23,142 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:23,143 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,143 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/453cfb5cdc8347dd86427552be555680, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=108.3 K 2024-12-08T04:01:23,143 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,143 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/453cfb5cdc8347dd86427552be555680, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8] 2024-12-08T04:01:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,143 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 453cfb5cdc8347dd86427552be555680, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733630475977 2024-12-08T04:01:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,143 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:23,143 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:23,143 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,143 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/2a20f7b260e44995a865a77ecb794599, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/e0541c4b7cb04bb484ad44c24cd34876, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/de54995b62854e3c80d7ababc15b25fc] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=35.9 K 2024-12-08T04:01:23,144 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd70845e4dab4173b08eee55c1187440, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733630477150 2024-12-08T04:01:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,144 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29cffe3c0452468489c27f08b23448a8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733630479328 2024-12-08T04:01:23,144 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a20f7b260e44995a865a77ecb794599, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733630475977 2024-12-08T04:01:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,145 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e0541c4b7cb04bb484ad44c24cd34876, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733630477150 2024-12-08T04:01:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,145 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting de54995b62854e3c80d7ababc15b25fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733630479330 2024-12-08T04:01:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,150 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,151 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412083779d3fee52a480e903615b787723345_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,152 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,153 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/8683d2501c8a4027941933a88e9e2c3f is 50, key is test_row_0/B:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,153 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412083779d3fee52a480e903615b787723345_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:23,153 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412083779d3fee52a480e903615b787723345_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742397_1573 (size=12595) 2024-12-08T04:01:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,160 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/8683d2501c8a4027941933a88e9e2c3f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8683d2501c8a4027941933a88e9e2c3f 2024-12-08T04:01:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,164 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into 8683d2501c8a4027941933a88e9e2c3f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:23,164 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:23,164 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=13, startTime=1733630483141; duration=0sec 2024-12-08T04:01:23,164 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:23,164 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:23,164 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,165 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:23,165 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:23,165 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,165 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/c1bee2df55b74af6af02cb745e4c5e32, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/51c71f095e334dc294c5c5ec8fe188e6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/94db48d553ec497b862e938753b736bb] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=35.9 K 2024-12-08T04:01:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,165 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting c1bee2df55b74af6af02cb745e4c5e32, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733630475977 2024-12-08T04:01:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,166 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c71f095e334dc294c5c5ec8fe188e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733630477150 2024-12-08T04:01:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,166 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 94db48d553ec497b862e938753b736bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733630479330 2024-12-08T04:01:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,173 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:23,173 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8234f6daabb34b30b9ad06b6c18f3e6d is 50, key is test_row_0/C:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742398_1574 (size=4469) 2024-12-08T04:01:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,176 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#485 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,176 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/98550e275ada4d23b7327729734ece4b is 175, key is test_row_0/A:col10/1733630481489/Put/seqid=0 2024-12-08T04:01:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742399_1575 (size=12595) 2024-12-08T04:01:23,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742400_1576 (size=31549) 2024-12-08T04:01:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,197 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/98550e275ada4d23b7327729734ece4b as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/98550e275ada4d23b7327729734ece4b 2024-12-08T04:01:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,203 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into 98550e275ada4d23b7327729734ece4b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:23,203 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:23,203 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=13, startTime=1733630483141; duration=0sec 2024-12-08T04:01:23,203 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:23,203 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-08T04:01:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,279 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:23,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ca8772f255ce4eabb8fab833bd8a0135_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630481496/Put/seqid=0 2024-12-08T04:01:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742401_1577 (size=9814) 2024-12-08T04:01:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:23,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630483562, max timestamp=1733630483574Num uniques 859; 2024-12-08T04:01:23,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630483562, max timestamp=1733630483574Num uniques 859; 2024-12-08T04:01:23,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:01:23,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:01:23,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733630483562, max timestamp=1733630483574Num uniques 859; 2024-12-08T04:01:23,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T04:01:23,595 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8234f6daabb34b30b9ad06b6c18f3e6d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8234f6daabb34b30b9ad06b6c18f3e6d 2024-12-08T04:01:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:23,600 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into 8234f6daabb34b30b9ad06b6c18f3e6d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:23,600 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:23,600 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=13, startTime=1733630483142; duration=0sec 2024-12-08T04:01:23,600 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:23,600 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:23,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630483609, max timestamp=1733630483617Num uniques 1068; 2024-12-08T04:01:23,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T04:01:23,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630483609, max timestamp=1733630483617Num uniques 1068; 2024-12-08T04:01:23,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733630483609, max timestamp=1733630483617Num uniques 1068; 2024-12-08T04:01:23,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T04:01:23,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T04:01:23,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733630483645, max timestamp=1733630483652Num uniques 1027; 2024-12-08T04:01:23,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T04:01:23,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733630483645, max timestamp=1733630483652Num uniques 1027; 2024-12-08T04:01:23,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T04:01:23,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733630483645, max timestamp=1733630483652Num uniques 1027; 2024-12-08T04:01:23,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T04:01:23,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733630483660, max timestamp=1733630483671Num uniques 1086; 2024-12-08T04:01:23,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-08T04:01:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733630483660, max timestamp=1733630483671Num uniques 1086; 2024-12-08T04:01:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-08T04:01:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733630483660, max timestamp=1733630483671Num uniques 1086; 2024-12-08T04:01:23,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-08T04:01:23,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733630483685, max timestamp=1733630483691Num uniques 1045; 2024-12-08T04:01:23,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733630483685, max timestamp=1733630483691Num uniques 1045; 2024-12-08T04:01:23,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-08T04:01:23,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-08T04:01:23,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733630483685, max timestamp=1733630483691Num uniques 1045; 2024-12-08T04:01:23,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-08T04:01:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,697 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ca8772f255ce4eabb8fab833bd8a0135_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ca8772f255ce4eabb8fab833bd8a0135_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:23,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e65a729d148d49ed9932215ad4f750f8, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e65a729d148d49ed9932215ad4f750f8 is 175, key is test_row_0/A:col10/1733630481496/Put/seqid=0 2024-12-08T04:01:23,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742402_1578 (size=22461) 2024-12-08T04:01:23,707 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e65a729d148d49ed9932215ad4f750f8 2024-12-08T04:01:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/7f0d6e1a7ad74dd8aca44c37e2f35623 is 50, key is test_row_0/B:col10/1733630481496/Put/seqid=0 2024-12-08T04:01:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630543714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630543714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630543715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733630483712, max timestamp=1733630483715Num uniques 1104; 2024-12-08T04:01:23,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733630483712, max timestamp=1733630483715Num uniques 1104; 2024-12-08T04:01:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T04:01:23,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T04:01:23,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733630483712, max timestamp=1733630483715Num uniques 1104; 2024-12-08T04:01:23,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-12-08T04:01:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742403_1579 (size=9757) 2024-12-08T04:01:23,726 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/7f0d6e1a7ad74dd8aca44c37e2f35623 2024-12-08T04:01:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/4934b725ab5949778561e900f7c89162 is 50, key is test_row_0/C:col10/1733630481496/Put/seqid=0 2024-12-08T04:01:23,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733630483747, max timestamp=1733630483760Num uniques 1013; 2024-12-08T04:01:23,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733630483747, max timestamp=1733630483760Num uniques 1013; 2024-12-08T04:01:23,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T04:01:23,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T04:01:23,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742404_1580 (size=9757) 2024-12-08T04:01:23,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733630483747, max timestamp=1733630483760Num uniques 1009; 2024-12-08T04:01:23,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/32df4b4bb439:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-08T04:01:23,766 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/4934b725ab5949778561e900f7c89162 2024-12-08T04:01:23,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e65a729d148d49ed9932215ad4f750f8 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8 2024-12-08T04:01:23,774 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8, entries=100, sequenceid=246, filesize=21.9 K 2024-12-08T04:01:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/7f0d6e1a7ad74dd8aca44c37e2f35623 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/7f0d6e1a7ad74dd8aca44c37e2f35623 2024-12-08T04:01:23,779 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/7f0d6e1a7ad74dd8aca44c37e2f35623, entries=100, sequenceid=246, filesize=9.5 K 2024-12-08T04:01:23,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/4934b725ab5949778561e900f7c89162 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4934b725ab5949778561e900f7c89162 2024-12-08T04:01:23,785 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4934b725ab5949778561e900f7c89162, entries=100, sequenceid=246, filesize=9.5 K 2024-12-08T04:01:23,785 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=100.06 KB/102462 for ca44a140da9733e9d0e45118f13fb6f0 in 506ms, sequenceid=246, compaction requested=false 2024-12-08T04:01:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-08T04:01:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-08T04:01:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:23,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=106.77 KB heapSize=303.60 KB 2024-12-08T04:01:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T04:01:23,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-08T04:01:23,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-12-08T04:01:23,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.3440 sec 2024-12-08T04:01:23,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082c1249c5176f44ee864094bacf1e9060_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:23,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742405_1581 (size=14994) 2024-12-08T04:01:23,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:23,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630543811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630543812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,816 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082c1249c5176f44ee864094bacf1e9060_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082c1249c5176f44ee864094bacf1e9060_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:23,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/ee62373cf3e54bbf9831b39b9f2b1b15, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:23,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/ee62373cf3e54bbf9831b39b9f2b1b15 is 175, key is test_row_0/A:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:23,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742406_1582 (size=39949) 2024-12-08T04:01:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630543827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630543827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630543827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630543927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630543927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630544037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630544037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630544037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630544137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630544137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,225 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/ee62373cf3e54bbf9831b39b9f2b1b15 2024-12-08T04:01:24,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/1d7d4531c663423689044ece0a4fc308 is 50, key is test_row_0/B:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:24,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742407_1583 (size=12301) 2024-12-08T04:01:24,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/1d7d4531c663423689044ece0a4fc308 2024-12-08T04:01:24,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/d4b6fd6fa6f249a99baae1529f794eb7 is 50, key is test_row_0/C:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:24,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742408_1584 (size=12301) 2024-12-08T04:01:24,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630544347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630544347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630544347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630544447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630544447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.21 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/d4b6fd6fa6f249a99baae1529f794eb7 2024-12-08T04:01:24,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/ee62373cf3e54bbf9831b39b9f2b1b15 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15 2024-12-08T04:01:24,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15, entries=200, sequenceid=383, filesize=39.0 K 2024-12-08T04:01:24,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/1d7d4531c663423689044ece0a4fc308 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/1d7d4531c663423689044ece0a4fc308 2024-12-08T04:01:24,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/1d7d4531c663423689044ece0a4fc308, entries=150, sequenceid=383, filesize=12.0 K 2024-12-08T04:01:24,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/d4b6fd6fa6f249a99baae1529f794eb7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d4b6fd6fa6f249a99baae1529f794eb7 2024-12-08T04:01:24,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d4b6fd6fa6f249a99baae1529f794eb7, entries=150, sequenceid=383, filesize=12.0 K 2024-12-08T04:01:24,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~106.77 KB/109332, heapSize ~303.55 KB/310840, currentSize=73.80 KB/75570 for ca44a140da9733e9d0e45118f13fb6f0 in 915ms, sequenceid=383, compaction requested=true 2024-12-08T04:01:24,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:24,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:24,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:24,703 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:24,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:24,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:24,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:24,704 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:24,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T04:01:24,704 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:24,704 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:24,705 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:24,705 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/98550e275ada4d23b7327729734ece4b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=91.8 K 2024-12-08T04:01:24,705 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:24,705 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/98550e275ada4d23b7327729734ece4b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15] 2024-12-08T04:01:24,705 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:24,705 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98550e275ada4d23b7327729734ece4b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733630479330 2024-12-08T04:01:24,705 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:24,705 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:24,705 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8683d2501c8a4027941933a88e9e2c3f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/7f0d6e1a7ad74dd8aca44c37e2f35623, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/1d7d4531c663423689044ece0a4fc308] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=33.8 K 2024-12-08T04:01:24,706 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65a729d148d49ed9932215ad4f750f8, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733630481496 2024-12-08T04:01:24,706 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8683d2501c8a4027941933a88e9e2c3f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733630479330 2024-12-08T04:01:24,706 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee62373cf3e54bbf9831b39b9f2b1b15, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733630483767 2024-12-08T04:01:24,706 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0d6e1a7ad74dd8aca44c37e2f35623, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733630481496 2024-12-08T04:01:24,706 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7d4531c663423689044ece0a4fc308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733630483767 2024-12-08T04:01:24,712 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:24,714 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:24,715 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/8ceb34b944c3470c8c97acb24b9b6e2d is 50, key is test_row_0/B:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:24,717 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208ac8e951443a34d549ef94a697f766624_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:24,719 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208ac8e951443a34d549ef94a697f766624_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:24,719 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ac8e951443a34d549ef94a697f766624_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:24,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742409_1585 (size=12847) 2024-12-08T04:01:24,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742410_1586 (size=4469) 2024-12-08T04:01:24,726 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/8ceb34b944c3470c8c97acb24b9b6e2d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8ceb34b944c3470c8c97acb24b9b6e2d 2024-12-08T04:01:24,730 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into 8ceb34b944c3470c8c97acb24b9b6e2d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:24,730 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:24,730 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=13, startTime=1733630484703; duration=0sec 2024-12-08T04:01:24,730 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:24,730 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:24,731 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:24,731 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:24,731 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:24,731 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:24,732 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8234f6daabb34b30b9ad06b6c18f3e6d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4934b725ab5949778561e900f7c89162, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d4b6fd6fa6f249a99baae1529f794eb7] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=33.8 K 2024-12-08T04:01:24,732 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8234f6daabb34b30b9ad06b6c18f3e6d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733630479330 2024-12-08T04:01:24,734 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 4934b725ab5949778561e900f7c89162, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733630481496 2024-12-08T04:01:24,734 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b6fd6fa6f249a99baae1529f794eb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733630483767 2024-12-08T04:01:24,739 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:24,739 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/24c9cfc3bc2f4581bac506884f6bc9b0 is 50, key is test_row_0/C:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:24,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742411_1587 (size=12847) 2024-12-08T04:01:24,746 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/24c9cfc3bc2f4581bac506884f6bc9b0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/24c9cfc3bc2f4581bac506884f6bc9b0 2024-12-08T04:01:24,749 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into 24c9cfc3bc2f4581bac506884f6bc9b0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:24,750 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:24,750 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=13, startTime=1733630484704; duration=0sec 2024-12-08T04:01:24,750 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:24,750 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:24,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T04:01:24,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:24,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:24,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:24,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:24,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:24,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:24,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120870d03538ed104779b6844b2f338acf9f_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630483810/Put/seqid=0 2024-12-08T04:01:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742412_1588 (size=12454) 2024-12-08T04:01:24,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:24,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120870d03538ed104779b6844b2f338acf9f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120870d03538ed104779b6844b2f338acf9f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:24,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/954e8699b96e46f38886fd23b4bbeaf7, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:24,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/954e8699b96e46f38886fd23b4bbeaf7 is 175, key is test_row_0/A:col10/1733630483810/Put/seqid=0 2024-12-08T04:01:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742413_1589 (size=31255) 2024-12-08T04:01:24,876 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/954e8699b96e46f38886fd23b4bbeaf7 2024-12-08T04:01:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630544878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630544878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630544879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f1378b9871e84d5ab353dd514070bc07 is 50, key is test_row_0/B:col10/1733630483810/Put/seqid=0 2024-12-08T04:01:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742414_1590 (size=12301) 2024-12-08T04:01:24,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f1378b9871e84d5ab353dd514070bc07 2024-12-08T04:01:24,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/1a3bcf420d8449d28f4c04a961f70046 is 50, key is test_row_0/C:col10/1733630483810/Put/seqid=0 2024-12-08T04:01:24,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742415_1591 (size=12301) 2024-12-08T04:01:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630544957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630544957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630544987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630544987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630544988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,125 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:25,125 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/116c8e248911455587a08fb0b2c7fb58 is 175, key is test_row_0/A:col10/1733630483783/Put/seqid=0 2024-12-08T04:01:25,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742416_1592 (size=31801) 2024-12-08T04:01:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630545197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630545197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630545197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/1a3bcf420d8449d28f4c04a961f70046 2024-12-08T04:01:25,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/954e8699b96e46f38886fd23b4bbeaf7 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7 2024-12-08T04:01:25,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7, entries=150, sequenceid=402, filesize=30.5 K 2024-12-08T04:01:25,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f1378b9871e84d5ab353dd514070bc07 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1378b9871e84d5ab353dd514070bc07 2024-12-08T04:01:25,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1378b9871e84d5ab353dd514070bc07, entries=150, sequenceid=402, filesize=12.0 K 2024-12-08T04:01:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/1a3bcf420d8449d28f4c04a961f70046 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/1a3bcf420d8449d28f4c04a961f70046 2024-12-08T04:01:25,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/1a3bcf420d8449d28f4c04a961f70046, entries=150, sequenceid=402, filesize=12.0 K 2024-12-08T04:01:25,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for ca44a140da9733e9d0e45118f13fb6f0 in 453ms, sequenceid=402, compaction requested=false 2024-12-08T04:01:25,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:25,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T04:01:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:25,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:25,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208608766336f914eb6bd9c3157f87ba21c_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:25,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630545521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630545522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742417_1593 (size=12454) 2024-12-08T04:01:25,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630545525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,531 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:25,534 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208608766336f914eb6bd9c3157f87ba21c_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208608766336f914eb6bd9c3157f87ba21c_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:25,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/4dc03ea360c2473a9a92bdc59010635c, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:25,535 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/116c8e248911455587a08fb0b2c7fb58 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/116c8e248911455587a08fb0b2c7fb58 2024-12-08T04:01:25,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/4dc03ea360c2473a9a92bdc59010635c is 175, key is test_row_0/A:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742418_1594 (size=31255) 2024-12-08T04:01:25,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/4dc03ea360c2473a9a92bdc59010635c 2024-12-08T04:01:25,540 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into 116c8e248911455587a08fb0b2c7fb58(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:25,540 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:25,540 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=13, startTime=1733630484703; duration=0sec 2024-12-08T04:01:25,540 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:25,540 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:25,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/9261d130d2c143e5b5cfda3f348d084c is 50, key is test_row_0/B:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742419_1595 (size=12301) 2024-12-08T04:01:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T04:01:25,617 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:25,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T04:01:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-08T04:01:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:25,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T04:01:25,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T04:01:25,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T04:01:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630545627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630545637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630545637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:25,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T04:01:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:25,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630545837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630545847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630545847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T04:01:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:25,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:25,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/9261d130d2c143e5b5cfda3f348d084c 2024-12-08T04:01:25,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/76e63af524314bc99f568db68ab39e43 is 50, key is test_row_0/C:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:25,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742420_1596 (size=12301) 2024-12-08T04:01:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630545967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630545967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:25,969 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,970 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T04:01:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:26,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630546147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630546157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630546157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T04:01:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:26,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/76e63af524314bc99f568db68ab39e43 2024-12-08T04:01:26,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/4dc03ea360c2473a9a92bdc59010635c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c 2024-12-08T04:01:26,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c, entries=150, sequenceid=420, filesize=30.5 K 2024-12-08T04:01:26,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/9261d130d2c143e5b5cfda3f348d084c as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/9261d130d2c143e5b5cfda3f348d084c 2024-12-08T04:01:26,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/9261d130d2c143e5b5cfda3f348d084c, entries=150, sequenceid=420, filesize=12.0 K 2024-12-08T04:01:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/76e63af524314bc99f568db68ab39e43 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/76e63af524314bc99f568db68ab39e43 2024-12-08T04:01:26,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/76e63af524314bc99f568db68ab39e43, entries=150, sequenceid=420, filesize=12.0 K 2024-12-08T04:01:26,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ca44a140da9733e9d0e45118f13fb6f0 in 872ms, sequenceid=420, compaction requested=true 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:26,381 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:26,381 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:26,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:26,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39919 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,382 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:26,382 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T04:01:26,382 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:26,382 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:26,382 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8ceb34b944c3470c8c97acb24b9b6e2d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1378b9871e84d5ab353dd514070bc07, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/9261d130d2c143e5b5cfda3f348d084c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=36.6 K 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:26,383 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,383 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/116c8e248911455587a08fb0b2c7fb58, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=92.1 K 2024-12-08T04:01:26,383 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/116c8e248911455587a08fb0b2c7fb58, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c] 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ceb34b944c3470c8c97acb24b9b6e2d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733630483767 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 116c8e248911455587a08fb0b2c7fb58, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733630483767 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954e8699b96e46f38886fd23b4bbeaf7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733630483810 2024-12-08T04:01:26,383 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f1378b9871e84d5ab353dd514070bc07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733630483810 2024-12-08T04:01:26,384 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc03ea360c2473a9a92bdc59010635c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733630484878 2024-12-08T04:01:26,384 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 9261d130d2c143e5b5cfda3f348d084c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733630484878 2024-12-08T04:01:26,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208951cdba01ed8400782063c118b462fa5_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630485521/Put/seqid=0 2024-12-08T04:01:26,405 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:26,407 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208007e97dbd0334df88db8011a274daa72_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:26,407 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:26,408 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f1ee66c0143f4f27ba393d6856b8a2df is 50, key is test_row_0/B:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:26,408 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208007e97dbd0334df88db8011a274daa72_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:26,408 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208007e97dbd0334df88db8011a274daa72_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:26,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742421_1597 (size=12949) 2024-12-08T04:01:26,417 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/f1ee66c0143f4f27ba393d6856b8a2df as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1ee66c0143f4f27ba393d6856b8a2df 2024-12-08T04:01:26,420 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into f1ee66c0143f4f27ba393d6856b8a2df(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:26,420 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:26,420 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=13, startTime=1733630486381; duration=0sec 2024-12-08T04:01:26,420 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:26,420 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:26,420 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:26,421 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:26,421 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:26,421 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:26,421 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/24c9cfc3bc2f4581bac506884f6bc9b0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/1a3bcf420d8449d28f4c04a961f70046, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/76e63af524314bc99f568db68ab39e43] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=36.6 K 2024-12-08T04:01:26,422 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c9cfc3bc2f4581bac506884f6bc9b0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733630483767 2024-12-08T04:01:26,422 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a3bcf420d8449d28f4c04a961f70046, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733630483810 2024-12-08T04:01:26,422 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e63af524314bc99f568db68ab39e43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733630484878 2024-12-08T04:01:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742422_1598 (size=4469) 2024-12-08T04:01:26,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742423_1599 (size=12454) 2024-12-08T04:01:26,436 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:26,437 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/e145bfa1709d4c6f8e857f2c8c80bc5d is 50, key is test_row_0/C:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:26,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742424_1600 (size=12949) 2024-12-08T04:01:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:26,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. as already flushing 2024-12-08T04:01:26,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630546681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630546682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630546685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630546797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630546797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630546797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:26,829 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:26,830 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/fcff58dc70a645dda16867b30fdf3a40 is 175, key is test_row_0/A:col10/1733630485509/Put/seqid=0 2024-12-08T04:01:26,832 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208951cdba01ed8400782063c118b462fa5_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208951cdba01ed8400782063c118b462fa5_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:26,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e295e95b83154ed782d3bda6063cebf0, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:26,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e295e95b83154ed782d3bda6063cebf0 is 175, key is test_row_0/A:col10/1733630485521/Put/seqid=0 2024-12-08T04:01:26,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742425_1601 (size=31903) 2024-12-08T04:01:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742426_1602 (size=31255) 2024-12-08T04:01:26,848 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/e145bfa1709d4c6f8e857f2c8c80bc5d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/e145bfa1709d4c6f8e857f2c8c80bc5d 2024-12-08T04:01:26,852 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into e145bfa1709d4c6f8e857f2c8c80bc5d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:26,852 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:26,852 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=13, startTime=1733630486381; duration=0sec 2024-12-08T04:01:26,852 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:26,852 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:26,889 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T04:01:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630547007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630547007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630547007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,239 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/fcff58dc70a645dda16867b30fdf3a40 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fcff58dc70a645dda16867b30fdf3a40 2024-12-08T04:01:27,240 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e295e95b83154ed782d3bda6063cebf0 2024-12-08T04:01:27,244 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into fcff58dc70a645dda16867b30fdf3a40(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:27,244 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:27,244 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=13, startTime=1733630486381; duration=0sec 2024-12-08T04:01:27,244 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:27,244 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:27,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/643ce2531a934174adc34cd6029009c5 is 50, key is test_row_0/B:col10/1733630485521/Put/seqid=0 2024-12-08T04:01:27,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742427_1603 (size=12301) 2024-12-08T04:01:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630547317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630547318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630547318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,649 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/643ce2531a934174adc34cd6029009c5 2024-12-08T04:01:27,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/ab6f3afb464b43a694cb44577feab665 is 50, key is test_row_0/C:col10/1733630485521/Put/seqid=0 2024-12-08T04:01:27,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742428_1604 (size=12301) 2024-12-08T04:01:27,675 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-12-08T04:01:27,675 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:27,675 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,675 INFO [Thread-2338 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2338. 2024-12-08T04:01:27,675 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:27,676 DEBUG [Thread-2338 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:27,676 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,676 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:27,676 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-12-08T04:01:27,676 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:27,676 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,677 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:27,678 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-12-08T04:01:27,678 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:27,678 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,678 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:27,679 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-12-08T04:01:27,679 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:27,679 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,679 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:27,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:27,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34950 deadline: 1733630547827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34900 deadline: 1733630547828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34878 deadline: 1733630547828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34940 deadline: 1733630547977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,978 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:27,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-08T04:01:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34916 deadline: 1733630547987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:27,987 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 32df4b4bb439,39919,1733630308479 for 'test_row_1' in ca44a140da9733e9d0e45118f13fb6f0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-08T04:01:27,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 , the old value is region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T04:01:27,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., hostname=32df4b4bb439,39919,1733630308479, seqNum=5 because the exception is null or not the one we care about 2024-12-08T04:01:28,061 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/ab6f3afb464b43a694cb44577feab665 2024-12-08T04:01:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/e295e95b83154ed782d3bda6063cebf0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0 2024-12-08T04:01:28,073 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0, entries=150, sequenceid=435, filesize=30.5 K 2024-12-08T04:01:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/643ce2531a934174adc34cd6029009c5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/643ce2531a934174adc34cd6029009c5 2024-12-08T04:01:28,077 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/643ce2531a934174adc34cd6029009c5, entries=150, sequenceid=435, filesize=12.0 K 2024-12-08T04:01:28,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/ab6f3afb464b43a694cb44577feab665 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/ab6f3afb464b43a694cb44577feab665 2024-12-08T04:01:28,081 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/ab6f3afb464b43a694cb44577feab665, entries=150, sequenceid=435, filesize=12.0 K 2024-12-08T04:01:28,081 INFO [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ca44a140da9733e9d0e45118f13fb6f0 in 1699ms, sequenceid=435, compaction requested=false 2024-12-08T04:01:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:28,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-08T04:01:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-08T04:01:28,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-08T04:01:28,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4630 sec 2024-12-08T04:01:28,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.4670 sec 2024-12-08T04:01:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] regionserver.HRegion(8855): Flush requested on ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:28,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T04:01:28,840 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-12-08T04:01:28,840 INFO [Thread-2327 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2327. 2024-12-08T04:01:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:28,840 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:28,840 DEBUG [Thread-2327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:28,840 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:28,840 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:28,840 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:28,840 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:28,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:28,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:28,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a56926ff1eb245f89be58a8cf4dcecfb_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_0/A:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:28,849 INFO [Thread-2329 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2329. 2024-12-08T04:01:28,849 DEBUG [Thread-2329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:28,849 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:28,849 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:28,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:28,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742429_1605 (size=12454) 2024-12-08T04:01:29,253 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:29,260 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208a56926ff1eb245f89be58a8cf4dcecfb_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a56926ff1eb245f89be58a8cf4dcecfb_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:29,261 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/113741f90756447dab45bc65d7f555ab, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:29,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/113741f90756447dab45bc65d7f555ab is 175, key is test_row_0/A:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:29,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742430_1606 (size=31255) 2024-12-08T04:01:29,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/113741f90756447dab45bc65d7f555ab 2024-12-08T04:01:29,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/ea91821dc03d4c8dabd3246533edd27a is 50, key is test_row_0/B:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:29,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742431_1607 (size=12301) 2024-12-08T04:01:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-08T04:01:29,778 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:30,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/ea91821dc03d4c8dabd3246533edd27a 2024-12-08T04:01:30,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8cc3c3ccef7541f894ea63915abfac28 is 50, key is test_row_0/C:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:30,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742432_1608 (size=12301) 2024-12-08T04:01:30,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8cc3c3ccef7541f894ea63915abfac28 2024-12-08T04:01:30,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/113741f90756447dab45bc65d7f555ab as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab 2024-12-08T04:01:30,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab, entries=150, sequenceid=458, filesize=30.5 K 2024-12-08T04:01:30,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/ea91821dc03d4c8dabd3246533edd27a as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ea91821dc03d4c8dabd3246533edd27a 2024-12-08T04:01:30,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ea91821dc03d4c8dabd3246533edd27a, entries=150, sequenceid=458, filesize=12.0 K 2024-12-08T04:01:30,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/8cc3c3ccef7541f894ea63915abfac28 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8cc3c3ccef7541f894ea63915abfac28 2024-12-08T04:01:30,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8cc3c3ccef7541f894ea63915abfac28, entries=150, sequenceid=458, filesize=12.0 K 2024-12-08T04:01:30,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for ca44a140da9733e9d0e45118f13fb6f0 in 1686ms, sequenceid=458, compaction requested=true 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T04:01:30,525 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca44a140da9733e9d0e45118f13fb6f0:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T04:01:30,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:30,525 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/A is initiating minor compaction (all files) 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/B is initiating minor compaction (all files) 2024-12-08T04:01:30,526 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/A in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:30,526 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/B in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:30,526 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fcff58dc70a645dda16867b30fdf3a40, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=92.2 K 2024-12-08T04:01:30,526 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1ee66c0143f4f27ba393d6856b8a2df, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/643ce2531a934174adc34cd6029009c5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ea91821dc03d4c8dabd3246533edd27a] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=36.7 K 2024-12-08T04:01:30,526 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. files: [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fcff58dc70a645dda16867b30fdf3a40, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab] 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcff58dc70a645dda16867b30fdf3a40, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733630484878 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ee66c0143f4f27ba393d6856b8a2df, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733630484878 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting e295e95b83154ed782d3bda6063cebf0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733630485519 2024-12-08T04:01:30,526 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 643ce2531a934174adc34cd6029009c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733630485519 2024-12-08T04:01:30,527 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ea91821dc03d4c8dabd3246533edd27a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733630486680 2024-12-08T04:01:30,527 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113741f90756447dab45bc65d7f555ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733630486680 2024-12-08T04:01:30,533 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:30,534 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:30,535 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/855b35a587374a7c9f5379af6eb278c0 is 50, key is test_row_0/B:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:30,535 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412085d337c30457d44c699c6404bd16ab2fd_ca44a140da9733e9d0e45118f13fb6f0 store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:30,538 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412085d337c30457d44c699c6404bd16ab2fd_ca44a140da9733e9d0e45118f13fb6f0, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:30,538 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085d337c30457d44c699c6404bd16ab2fd_ca44a140da9733e9d0e45118f13fb6f0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:30,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742433_1609 (size=13051) 2024-12-08T04:01:30,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742434_1610 (size=4469) 2024-12-08T04:01:30,943 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:30,944 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/7dac975185334784aed616344af5a5c5 is 175, key is test_row_0/A:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:30,949 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/855b35a587374a7c9f5379af6eb278c0 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/855b35a587374a7c9f5379af6eb278c0 2024-12-08T04:01:30,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742435_1611 (size=32005) 2024-12-08T04:01:30,956 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/B of ca44a140da9733e9d0e45118f13fb6f0 into 855b35a587374a7c9f5379af6eb278c0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:30,956 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:30,956 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/B, priority=13, startTime=1733630490525; duration=0sec 2024-12-08T04:01:30,956 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T04:01:30,956 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:B 2024-12-08T04:01:30,956 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T04:01:30,957 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T04:01:30,957 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1541): ca44a140da9733e9d0e45118f13fb6f0/C is initiating minor compaction (all files) 2024-12-08T04:01:30,957 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca44a140da9733e9d0e45118f13fb6f0/C in TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:30,957 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/e145bfa1709d4c6f8e857f2c8c80bc5d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/ab6f3afb464b43a694cb44577feab665, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8cc3c3ccef7541f894ea63915abfac28] into tmpdir=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp, totalSize=36.7 K 2024-12-08T04:01:30,957 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting e145bfa1709d4c6f8e857f2c8c80bc5d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733630484878 2024-12-08T04:01:30,958 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6f3afb464b43a694cb44577feab665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1733630485519 2024-12-08T04:01:30,958 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc3c3ccef7541f894ea63915abfac28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733630486680 2024-12-08T04:01:30,965 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca44a140da9733e9d0e45118f13fb6f0#C#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T04:01:30,966 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/54a6f674ae8d4ec2ac855a7416931f5f is 50, key is test_row_0/C:col10/1733630486680/Put/seqid=0 2024-12-08T04:01:30,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742436_1612 (size=13051) 2024-12-08T04:01:31,353 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/7dac975185334784aed616344af5a5c5 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/7dac975185334784aed616344af5a5c5 2024-12-08T04:01:31,357 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/A of ca44a140da9733e9d0e45118f13fb6f0 into 7dac975185334784aed616344af5a5c5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:31,357 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:31,357 INFO [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/A, priority=13, startTime=1733630490525; duration=0sec 2024-12-08T04:01:31,357 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:31,357 DEBUG [RS:0;32df4b4bb439:39919-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:A 2024-12-08T04:01:31,373 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/54a6f674ae8d4ec2ac855a7416931f5f as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/54a6f674ae8d4ec2ac855a7416931f5f 2024-12-08T04:01:31,378 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca44a140da9733e9d0e45118f13fb6f0/C of ca44a140da9733e9d0e45118f13fb6f0 into 54a6f674ae8d4ec2ac855a7416931f5f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T04:01:31,378 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca44a140da9733e9d0e45118f13fb6f0: 2024-12-08T04:01:31,378 INFO [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., storeName=ca44a140da9733e9d0e45118f13fb6f0/C, priority=13, startTime=1733630490525; duration=0sec 2024-12-08T04:01:31,378 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T04:01:31,378 DEBUG [RS:0;32df4b4bb439:39919-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca44a140da9733e9d0e45118f13fb6f0:C 2024-12-08T04:01:32,021 INFO [Thread-2331 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2331. 2024-12-08T04:01:32,021 DEBUG [Thread-2331 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:32,021 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:32,022 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:32,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:32,031 INFO [Thread-2325 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2325. 2024-12-08T04:01:32,031 DEBUG [Thread-2325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T04:01:32,031 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:32,031 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:32,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4486 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4469 2024-12-08T04:01:32,031 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4395 2024-12-08T04:01:32,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4472 2024-12-08T04:01:32,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4442 2024-12-08T04:01:32,032 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T04:01:32,032 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:32,032 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:32,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:32,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:32,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:32,033 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T04:01:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T04:01:32,037 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630492036"}]},"ts":"1733630492036"} 2024-12-08T04:01:32,038 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T04:01:32,038 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T04:01:32,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T04:01:32,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, UNASSIGN}] 2024-12-08T04:01:32,040 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, UNASSIGN 2024-12-08T04:01:32,041 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=CLOSING, regionLocation=32df4b4bb439,39919,1733630308479 2024-12-08T04:01:32,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, UNASSIGN because future has completed 2024-12-08T04:01:32,042 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T04:01:32,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479}] 2024-12-08T04:01:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T04:01:32,194 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:32,194 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T04:01:32,194 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing ca44a140da9733e9d0e45118f13fb6f0, disabling compactions & flushes 2024-12-08T04:01:32,194 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. after waiting 0 ms 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:32,195 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing ca44a140da9733e9d0e45118f13fb6f0 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=A 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=B 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca44a140da9733e9d0e45118f13fb6f0, store=C 2024-12-08T04:01:32,195 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T04:01:32,202 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082d7088186bc74a30a5de5ee529905950_ca44a140da9733e9d0e45118f13fb6f0 is 50, key is test_row_1/A:col10/1733630492029/Put/seqid=0 2024-12-08T04:01:32,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742437_1613 (size=7374) 2024-12-08T04:01:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T04:01:32,608 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:32,616 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082d7088186bc74a30a5de5ee529905950_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082d7088186bc74a30a5de5ee529905950_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:32,617 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/0f8500f523304adcbda476f20d11d6ca, store: [table=TestAcidGuarantees family=A region=ca44a140da9733e9d0e45118f13fb6f0] 2024-12-08T04:01:32,618 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/0f8500f523304adcbda476f20d11d6ca is 175, key is test_row_1/A:col10/1733630492029/Put/seqid=0 2024-12-08T04:01:32,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742438_1614 (size=13865) 2024-12-08T04:01:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T04:01:33,023 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=467, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/0f8500f523304adcbda476f20d11d6ca 2024-12-08T04:01:33,034 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/4f8a49b4cd114e6399b5e8604c580a29 is 50, key is test_row_1/B:col10/1733630492029/Put/seqid=0 2024-12-08T04:01:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742439_1615 (size=7415) 2024-12-08T04:01:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T04:01:33,440 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/4f8a49b4cd114e6399b5e8604c580a29 2024-12-08T04:01:33,451 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/fd4e2f1418054ea894c148e961f423e9 is 50, key is test_row_1/C:col10/1733630492029/Put/seqid=0 2024-12-08T04:01:33,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742440_1616 (size=7415) 2024-12-08T04:01:33,856 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/fd4e2f1418054ea894c148e961f423e9 2024-12-08T04:01:33,865 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/A/0f8500f523304adcbda476f20d11d6ca as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0f8500f523304adcbda476f20d11d6ca 2024-12-08T04:01:33,871 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0f8500f523304adcbda476f20d11d6ca, entries=50, sequenceid=467, filesize=13.5 K 2024-12-08T04:01:33,871 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/B/4f8a49b4cd114e6399b5e8604c580a29 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/4f8a49b4cd114e6399b5e8604c580a29 2024-12-08T04:01:33,876 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/4f8a49b4cd114e6399b5e8604c580a29, entries=50, sequenceid=467, filesize=7.2 K 2024-12-08T04:01:33,876 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/.tmp/C/fd4e2f1418054ea894c148e961f423e9 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/fd4e2f1418054ea894c148e961f423e9 2024-12-08T04:01:33,880 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/fd4e2f1418054ea894c148e961f423e9, entries=50, sequenceid=467, filesize=7.2 K 2024-12-08T04:01:33,881 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ca44a140da9733e9d0e45118f13fb6f0 in 1686ms, sequenceid=467, compaction requested=false 2024-12-08T04:01:33,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/5d6fa5d4fd194f9889f29101d259596e, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0c8a8eb89ea14fa086b60e4ba90336f6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/453cfb5cdc8347dd86427552be555680, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/98550e275ada4d23b7327729734ece4b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/116c8e248911455587a08fb0b2c7fb58, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fcff58dc70a645dda16867b30fdf3a40, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab] to archive 2024-12-08T04:01:33,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:01:33,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/f8d98b2790f641feb314258310a20cba 2024-12-08T04:01:33,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/80cda37ca2a8473eae13aa0131e9657e 2024-12-08T04:01:33,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/6a0d1a4537054aedaecb3de95a06a5b1 2024-12-08T04:01:33,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/5d6fa5d4fd194f9889f29101d259596e to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/5d6fa5d4fd194f9889f29101d259596e 2024-12-08T04:01:33,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/896f4f890c7f47e7986d5ecf4161f386 2024-12-08T04:01:33,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/59c4b2b9b8ab4248a9e5b632a042f1e8 2024-12-08T04:01:33,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0c8a8eb89ea14fa086b60e4ba90336f6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0c8a8eb89ea14fa086b60e4ba90336f6 2024-12-08T04:01:33,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/a355fafcb02740dd9f9cba84d25c91fc 2024-12-08T04:01:33,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/9cd05e7292504a9b83d187b41b0e1a41 2024-12-08T04:01:33,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/401c1f1653bc438da17a6bedaa508709 2024-12-08T04:01:33,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/453cfb5cdc8347dd86427552be555680 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/453cfb5cdc8347dd86427552be555680 2024-12-08T04:01:33,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fd70845e4dab4173b08eee55c1187440 2024-12-08T04:01:33,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/29cffe3c0452468489c27f08b23448a8 2024-12-08T04:01:33,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/98550e275ada4d23b7327729734ece4b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/98550e275ada4d23b7327729734ece4b 2024-12-08T04:01:33,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e65a729d148d49ed9932215ad4f750f8 2024-12-08T04:01:33,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/ee62373cf3e54bbf9831b39b9f2b1b15 2024-12-08T04:01:33,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/116c8e248911455587a08fb0b2c7fb58 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/116c8e248911455587a08fb0b2c7fb58 2024-12-08T04:01:33,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/954e8699b96e46f38886fd23b4bbeaf7 2024-12-08T04:01:33,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fcff58dc70a645dda16867b30fdf3a40 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/fcff58dc70a645dda16867b30fdf3a40 2024-12-08T04:01:33,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/4dc03ea360c2473a9a92bdc59010635c 2024-12-08T04:01:33,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/e295e95b83154ed782d3bda6063cebf0 2024-12-08T04:01:33,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/113741f90756447dab45bc65d7f555ab 2024-12-08T04:01:33,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/dd68a472f8da4894967a4c1d81b2cff9, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/c482e20a993246ff9db451b5dedd6fce, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f62e77cdf81d4ad1b233fbfb06d32ad3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/a8b8330231764036862dda209937aa8b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/895459cfde264e2abc5ff790125cb876, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ac1f7c518b0b49f5a228e5982b0e4cd0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/75265b7e7a76441a9899e9716356cd69, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d5e9efc2078c4c64861b854e491c2c3b, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d89b376c6e624135b50ab8a59d666f31, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/2a20f7b260e44995a865a77ecb794599, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/cd591f46d6b6441ab8d3c8040fc6def5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/e0541c4b7cb04bb484ad44c24cd34876, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8683d2501c8a4027941933a88e9e2c3f, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/de54995b62854e3c80d7ababc15b25fc, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/7f0d6e1a7ad74dd8aca44c37e2f35623, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8ceb34b944c3470c8c97acb24b9b6e2d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/1d7d4531c663423689044ece0a4fc308, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1378b9871e84d5ab353dd514070bc07, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1ee66c0143f4f27ba393d6856b8a2df, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/9261d130d2c143e5b5cfda3f348d084c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/643ce2531a934174adc34cd6029009c5, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ea91821dc03d4c8dabd3246533edd27a] to archive 2024-12-08T04:01:33,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:01:33,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/dd68a472f8da4894967a4c1d81b2cff9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/dd68a472f8da4894967a4c1d81b2cff9 2024-12-08T04:01:33,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/c482e20a993246ff9db451b5dedd6fce to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/c482e20a993246ff9db451b5dedd6fce 2024-12-08T04:01:33,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f62e77cdf81d4ad1b233fbfb06d32ad3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f62e77cdf81d4ad1b233fbfb06d32ad3 2024-12-08T04:01:33,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/a8b8330231764036862dda209937aa8b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/a8b8330231764036862dda209937aa8b 2024-12-08T04:01:33,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/895459cfde264e2abc5ff790125cb876 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/895459cfde264e2abc5ff790125cb876 2024-12-08T04:01:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ac1f7c518b0b49f5a228e5982b0e4cd0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ac1f7c518b0b49f5a228e5982b0e4cd0 2024-12-08T04:01:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/75265b7e7a76441a9899e9716356cd69 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/75265b7e7a76441a9899e9716356cd69 2024-12-08T04:01:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d5e9efc2078c4c64861b854e491c2c3b to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d5e9efc2078c4c64861b854e491c2c3b 2024-12-08T04:01:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d89b376c6e624135b50ab8a59d666f31 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/d89b376c6e624135b50ab8a59d666f31 2024-12-08T04:01:33,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/2a20f7b260e44995a865a77ecb794599 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/2a20f7b260e44995a865a77ecb794599 2024-12-08T04:01:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/cd591f46d6b6441ab8d3c8040fc6def5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/cd591f46d6b6441ab8d3c8040fc6def5 2024-12-08T04:01:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/e0541c4b7cb04bb484ad44c24cd34876 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/e0541c4b7cb04bb484ad44c24cd34876 2024-12-08T04:01:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8683d2501c8a4027941933a88e9e2c3f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8683d2501c8a4027941933a88e9e2c3f 2024-12-08T04:01:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/de54995b62854e3c80d7ababc15b25fc to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/de54995b62854e3c80d7ababc15b25fc 2024-12-08T04:01:33,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/7f0d6e1a7ad74dd8aca44c37e2f35623 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/7f0d6e1a7ad74dd8aca44c37e2f35623 2024-12-08T04:01:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8ceb34b944c3470c8c97acb24b9b6e2d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/8ceb34b944c3470c8c97acb24b9b6e2d 2024-12-08T04:01:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/1d7d4531c663423689044ece0a4fc308 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/1d7d4531c663423689044ece0a4fc308 2024-12-08T04:01:33,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1378b9871e84d5ab353dd514070bc07 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1378b9871e84d5ab353dd514070bc07 2024-12-08T04:01:33,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1ee66c0143f4f27ba393d6856b8a2df to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/f1ee66c0143f4f27ba393d6856b8a2df 2024-12-08T04:01:33,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/9261d130d2c143e5b5cfda3f348d084c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/9261d130d2c143e5b5cfda3f348d084c 2024-12-08T04:01:33,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/643ce2531a934174adc34cd6029009c5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/643ce2531a934174adc34cd6029009c5 2024-12-08T04:01:33,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ea91821dc03d4c8dabd3246533edd27a to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/ea91821dc03d4c8dabd3246533edd27a 2024-12-08T04:01:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/0b5f468c058341f4829aeae561d20c41, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/b6daee9c378649588df5de2b21c4fe00, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d187185e34f44510ba894f98541084b4, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/6c5cef8524ee409790362e1334bf8829, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a1279ad0389943128f47c6db5f2118ef, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a86ddc3a781141559e2ce0c0ebbc6e9c, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/5f7cd811e3614fd9a328c91c67405613, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8792e3803e23436b84e991c3d6518c91, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4b3d704038184f82a6aca7e3ed9f6dc1, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/c1bee2df55b74af6af02cb745e4c5e32, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/cb0cf26c11154bd1b63cedf62bf4fab3, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/51c71f095e334dc294c5c5ec8fe188e6, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8234f6daabb34b30b9ad06b6c18f3e6d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/94db48d553ec497b862e938753b736bb, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4934b725ab5949778561e900f7c89162, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/24c9cfc3bc2f4581bac506884f6bc9b0, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d4b6fd6fa6f249a99baae1529f794eb7, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/1a3bcf420d8449d28f4c04a961f70046, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/e145bfa1709d4c6f8e857f2c8c80bc5d, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/76e63af524314bc99f568db68ab39e43, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/ab6f3afb464b43a694cb44577feab665, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8cc3c3ccef7541f894ea63915abfac28] to archive 2024-12-08T04:01:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T04:01:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/0b5f468c058341f4829aeae561d20c41 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/0b5f468c058341f4829aeae561d20c41 2024-12-08T04:01:33,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/b6daee9c378649588df5de2b21c4fe00 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/b6daee9c378649588df5de2b21c4fe00 2024-12-08T04:01:33,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d187185e34f44510ba894f98541084b4 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d187185e34f44510ba894f98541084b4 2024-12-08T04:01:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/6c5cef8524ee409790362e1334bf8829 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/6c5cef8524ee409790362e1334bf8829 2024-12-08T04:01:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a1279ad0389943128f47c6db5f2118ef to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a1279ad0389943128f47c6db5f2118ef 2024-12-08T04:01:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a86ddc3a781141559e2ce0c0ebbc6e9c to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/a86ddc3a781141559e2ce0c0ebbc6e9c 2024-12-08T04:01:33,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/5f7cd811e3614fd9a328c91c67405613 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/5f7cd811e3614fd9a328c91c67405613 2024-12-08T04:01:33,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8792e3803e23436b84e991c3d6518c91 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8792e3803e23436b84e991c3d6518c91 2024-12-08T04:01:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4b3d704038184f82a6aca7e3ed9f6dc1 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4b3d704038184f82a6aca7e3ed9f6dc1 2024-12-08T04:01:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/c1bee2df55b74af6af02cb745e4c5e32 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/c1bee2df55b74af6af02cb745e4c5e32 2024-12-08T04:01:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/cb0cf26c11154bd1b63cedf62bf4fab3 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/cb0cf26c11154bd1b63cedf62bf4fab3 2024-12-08T04:01:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/51c71f095e334dc294c5c5ec8fe188e6 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/51c71f095e334dc294c5c5ec8fe188e6 2024-12-08T04:01:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8234f6daabb34b30b9ad06b6c18f3e6d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8234f6daabb34b30b9ad06b6c18f3e6d 2024-12-08T04:01:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/94db48d553ec497b862e938753b736bb to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/94db48d553ec497b862e938753b736bb 2024-12-08T04:01:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4934b725ab5949778561e900f7c89162 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/4934b725ab5949778561e900f7c89162 2024-12-08T04:01:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/24c9cfc3bc2f4581bac506884f6bc9b0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/24c9cfc3bc2f4581bac506884f6bc9b0 2024-12-08T04:01:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d4b6fd6fa6f249a99baae1529f794eb7 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/d4b6fd6fa6f249a99baae1529f794eb7 2024-12-08T04:01:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/1a3bcf420d8449d28f4c04a961f70046 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/1a3bcf420d8449d28f4c04a961f70046 2024-12-08T04:01:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/e145bfa1709d4c6f8e857f2c8c80bc5d to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/e145bfa1709d4c6f8e857f2c8c80bc5d 2024-12-08T04:01:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/76e63af524314bc99f568db68ab39e43 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/76e63af524314bc99f568db68ab39e43 2024-12-08T04:01:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/ab6f3afb464b43a694cb44577feab665 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/ab6f3afb464b43a694cb44577feab665 2024-12-08T04:01:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8cc3c3ccef7541f894ea63915abfac28 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/8cc3c3ccef7541f894ea63915abfac28 2024-12-08T04:01:33,938 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/recovered.edits/470.seqid, newMaxSeqId=470, maxSeqId=4 2024-12-08T04:01:33,939 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. 2024-12-08T04:01:33,939 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for ca44a140da9733e9d0e45118f13fb6f0: Waiting for close lock at 1733630492194Running coprocessor pre-close hooks at 1733630492194Disabling compacts and flushes for region at 1733630492194Disabling writes for close at 1733630492195 (+1 ms)Obtaining lock to block concurrent updates at 1733630492195Preparing flush snapshotting stores in ca44a140da9733e9d0e45118f13fb6f0 at 1733630492195Finished memstore snapshotting TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733630492195Flushing stores of TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0. at 1733630492196 (+1 ms)Flushing ca44a140da9733e9d0e45118f13fb6f0/A: creating writer at 1733630492196Flushing mob file ca44a140da9733e9d0e45118f13fb6f0/A: appending metadata at 1733630492202 (+6 ms)Flushing mob file ca44a140da9733e9d0e45118f13fb6f0/A: closing flushed file at 1733630492202Flushing ca44a140da9733e9d0e45118f13fb6f0/A: appending metadata at 1733630492617 (+415 ms)Flushing ca44a140da9733e9d0e45118f13fb6f0/A: closing flushed file at 1733630492617Flushing ca44a140da9733e9d0e45118f13fb6f0/B: creating writer at 1733630493031 (+414 ms)Flushing ca44a140da9733e9d0e45118f13fb6f0/B: appending metadata at 1733630493034 (+3 ms)Flushing ca44a140da9733e9d0e45118f13fb6f0/B: closing flushed file at 1733630493034Flushing ca44a140da9733e9d0e45118f13fb6f0/C: creating writer at 1733630493448 (+414 ms)Flushing ca44a140da9733e9d0e45118f13fb6f0/C: appending metadata at 1733630493451 (+3 ms)Flushing ca44a140da9733e9d0e45118f13fb6f0/C: closing flushed file at 1733630493451Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e32e0d: reopening flushed file at 1733630493864 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b4cc45e: reopening flushed file at 1733630493871 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b84bc08: reopening flushed file at 1733630493876 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ca44a140da9733e9d0e45118f13fb6f0 in 1686ms, sequenceid=467, compaction requested=false at 1733630493881 (+5 ms)Writing region close event to WAL at 1733630493936 (+55 ms)Running coprocessor post-close hooks at 1733630493939 (+3 ms)Closed at 1733630493939 2024-12-08T04:01:33,940 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:33,940 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=ca44a140da9733e9d0e45118f13fb6f0, regionState=CLOSED 2024-12-08T04:01:33,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 because future has completed 2024-12-08T04:01:33,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-08T04:01:33,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure ca44a140da9733e9d0e45118f13fb6f0, server=32df4b4bb439,39919,1733630308479 in 1.9000 sec 2024-12-08T04:01:33,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-12-08T04:01:33,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca44a140da9733e9d0e45118f13fb6f0, UNASSIGN in 1.9030 sec 2024-12-08T04:01:33,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-08T04:01:33,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9070 sec 2024-12-08T04:01:33,947 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630493947"}]},"ts":"1733630493947"} 2024-12-08T04:01:33,949 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T04:01:33,949 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T04:01:33,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-12-08T04:01:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T04:01:34,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:34,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T04:01:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:34,191 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-08T04:01:34,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:34,195 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,199 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C, FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/recovered.edits] 2024-12-08T04:01:34,204 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0f8500f523304adcbda476f20d11d6ca to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/0f8500f523304adcbda476f20d11d6ca 2024-12-08T04:01:34,206 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/7dac975185334784aed616344af5a5c5 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/A/7dac975185334784aed616344af5a5c5 2024-12-08T04:01:34,210 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/4f8a49b4cd114e6399b5e8604c580a29 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/4f8a49b4cd114e6399b5e8604c580a29 2024-12-08T04:01:34,212 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/855b35a587374a7c9f5379af6eb278c0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/B/855b35a587374a7c9f5379af6eb278c0 2024-12-08T04:01:34,216 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/54a6f674ae8d4ec2ac855a7416931f5f to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/54a6f674ae8d4ec2ac855a7416931f5f 2024-12-08T04:01:34,217 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/fd4e2f1418054ea894c148e961f423e9 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/C/fd4e2f1418054ea894c148e961f423e9 2024-12-08T04:01:34,220 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/recovered.edits/470.seqid to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0/recovered.edits/470.seqid 2024-12-08T04:01:34,220 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/default/TestAcidGuarantees/ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,221 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T04:01:34,221 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T04:01:34,221 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T04:01:34,224 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080015ef19b7e24b8badeb372ac037bcdb_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080015ef19b7e24b8badeb372ac037bcdb_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,225 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082056da7cfc434e43b0b21bb502f63a2c_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082056da7cfc434e43b0b21bb502f63a2c_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,226 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208280a247fa3cc4e98bb30e462ff896128_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208280a247fa3cc4e98bb30e462ff896128_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,226 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082c1249c5176f44ee864094bacf1e9060_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082c1249c5176f44ee864094bacf1e9060_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,227 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082d7088186bc74a30a5de5ee529905950_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082d7088186bc74a30a5de5ee529905950_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,228 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208608766336f914eb6bd9c3157f87ba21c_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208608766336f914eb6bd9c3157f87ba21c_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,229 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120867b1f076649f480198a684449993ef7f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120867b1f076649f480198a684449993ef7f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,230 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120870d03538ed104779b6844b2f338acf9f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120870d03538ed104779b6844b2f338acf9f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,231 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208951cdba01ed8400782063c118b462fa5_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208951cdba01ed8400782063c118b462fa5_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,232 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a56926ff1eb245f89be58a8cf4dcecfb_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a56926ff1eb245f89be58a8cf4dcecfb_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,232 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a6f8a24c420a4d77811f303e783a8a4f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208a6f8a24c420a4d77811f303e783a8a4f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,233 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208af33614376e1485aa18f442841d98d5f_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208af33614376e1485aa18f442841d98d5f_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,234 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208be90d076c2f24c1fbf4ce412cc7db006_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208be90d076c2f24c1fbf4ce412cc7db006_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,235 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ca8772f255ce4eabb8fab833bd8a0135_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ca8772f255ce4eabb8fab833bd8a0135_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,236 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d475a53119d04e98a429e80caadf98e4_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d475a53119d04e98a429e80caadf98e4_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,236 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e1c596c6b0924f30bce7378674547839_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e1c596c6b0924f30bce7378674547839_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,237 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ec9c2d3c805449679d9f6fff62d700ad_ca44a140da9733e9d0e45118f13fb6f0 to hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ec9c2d3c805449679d9f6fff62d700ad_ca44a140da9733e9d0e45118f13fb6f0 2024-12-08T04:01:34,237 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T04:01:34,239 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:34,241 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T04:01:34,243 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T04:01:34,244 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:34,244 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T04:01:34,244 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733630494244"}]},"ts":"9223372036854775807"} 2024-12-08T04:01:34,246 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T04:01:34,246 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ca44a140da9733e9d0e45118f13fb6f0, NAME => 'TestAcidGuarantees,,1733630464529.ca44a140da9733e9d0e45118f13fb6f0.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T04:01:34,246 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T04:01:34,247 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733630494247"}]},"ts":"9223372036854775807"} 2024-12-08T04:01:34,249 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T04:01:34,250 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T04:01:34,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-12-08T04:01:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-08T04:01:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T04:01:34,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T04:01:34,310 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=455 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=405 (was 419), ProcessCount=11 (was 11), AvailableMemoryMB=16740 (was 16944) 2024-12-08T04:01:34,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T04:01:34,311 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:34,311 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:34,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:34,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:34,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:34,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T04:01:34,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2038551008, stopped=false 2024-12-08T04:01:34,312 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32df4b4bb439,38065,1733630307764 2024-12-08T04:01:34,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:34,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:34,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:34,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:34,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T04:01:34,370 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:34,370 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:34,370 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:34,371 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:34,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:34,371 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,39919,1733630308479' ***** 2024-12-08T04:01:34,371 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T04:01:34,372 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T04:01:34,372 INFO [RS:0;32df4b4bb439:39919 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T04:01:34,373 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T04:01:34,373 INFO [RS:0;32df4b4bb439:39919 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T04:01:34,373 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,39919,1733630308479 2024-12-08T04:01:34,373 INFO [RS:0;32df4b4bb439:39919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:34,373 INFO [RS:0;32df4b4bb439:39919 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32df4b4bb439:39919. 2024-12-08T04:01:34,373 DEBUG [RS:0;32df4b4bb439:39919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:34,373 DEBUG [RS:0;32df4b4bb439:39919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:34,374 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T04:01:34,374 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T04:01:34,374 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T04:01:34,374 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T04:01:34,374 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T04:01:34,375 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T04:01:34,375 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:34,375 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T04:01:34,375 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T04:01:34,375 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T04:01:34,375 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T04:01:34,375 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T04:01:34,375 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-08T04:01:34,388 INFO [regionserver/32df4b4bb439:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T04:01:34,388 INFO [regionserver/32df4b4bb439:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T04:01:34,396 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/info/c864f50a2b1a47c7ad24a1dbea67403d is 95, key is TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac./info:/1733630340097/DeleteFamily/seqid=0 2024-12-08T04:01:34,397 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:34,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742441_1617 (size=6514) 2024-12-08T04:01:34,575 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:34,775 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:34,800 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/info/c864f50a2b1a47c7ad24a1dbea67403d 2024-12-08T04:01:34,824 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/ns/2d580217df7c4bd7a4df3acb0f334743 is 93, key is TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac./ns:/1733630340063/DeleteFamily/seqid=0 2024-12-08T04:01:34,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742442_1618 (size=5906) 2024-12-08T04:01:34,976 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:35,176 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:35,228 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/ns/2d580217df7c4bd7a4df3acb0f334743 2024-12-08T04:01:35,253 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/rep_barrier/50ce73eb58cc4a9bbcc9ee56ee0b7400 is 102, key is TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac./rep_barrier:/1733630340063/DeleteFamily/seqid=0 2024-12-08T04:01:35,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742443_1619 (size=6025) 2024-12-08T04:01:35,376 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:35,577 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:35,657 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/rep_barrier/50ce73eb58cc4a9bbcc9ee56ee0b7400 2024-12-08T04:01:35,683 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/table/eb4ec8173f87459599606cffa66086bd is 96, key is TestAcidGuarantees,,1733630312988.97ee24ee195dd7bff50b82d61598e8ac./table:/1733630340063/DeleteFamily/seqid=0 2024-12-08T04:01:35,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742444_1620 (size=5936) 2024-12-08T04:01:35,777 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:35,977 DEBUG [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T04:01:36,086 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/table/eb4ec8173f87459599606cffa66086bd 2024-12-08T04:01:36,091 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/info/c864f50a2b1a47c7ad24a1dbea67403d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/info/c864f50a2b1a47c7ad24a1dbea67403d 2024-12-08T04:01:36,095 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/info/c864f50a2b1a47c7ad24a1dbea67403d, entries=12, sequenceid=90, filesize=6.4 K 2024-12-08T04:01:36,096 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/ns/2d580217df7c4bd7a4df3acb0f334743 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/ns/2d580217df7c4bd7a4df3acb0f334743 2024-12-08T04:01:36,099 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/ns/2d580217df7c4bd7a4df3acb0f334743, entries=8, sequenceid=90, filesize=5.8 K 2024-12-08T04:01:36,100 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/rep_barrier/50ce73eb58cc4a9bbcc9ee56ee0b7400 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/rep_barrier/50ce73eb58cc4a9bbcc9ee56ee0b7400 2024-12-08T04:01:36,102 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/rep_barrier/50ce73eb58cc4a9bbcc9ee56ee0b7400, entries=6, sequenceid=90, filesize=5.9 K 2024-12-08T04:01:36,103 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/.tmp/table/eb4ec8173f87459599606cffa66086bd as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/table/eb4ec8173f87459599606cffa66086bd 2024-12-08T04:01:36,106 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/table/eb4ec8173f87459599606cffa66086bd, entries=7, sequenceid=90, filesize=5.8 K 2024-12-08T04:01:36,107 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false 2024-12-08T04:01:36,116 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-08T04:01:36,117 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T04:01:36,117 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:36,117 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630494375Running coprocessor pre-close hooks at 1733630494375Disabling compacts and flushes for region at 1733630494375Disabling writes for close at 1733630494375Obtaining lock to block concurrent updates at 1733630494375Preparing flush snapshotting stores in 1588230740 at 1733630494375Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733630494376 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733630494376Flushing 1588230740/info: creating writer at 1733630494377 (+1 ms)Flushing 1588230740/info: appending metadata at 1733630494395 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733630494395Flushing 1588230740/ns: creating writer at 1733630494809 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733630494824 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733630494824Flushing 1588230740/rep_barrier: creating writer at 1733630495235 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733630495253 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733630495253Flushing 1588230740/table: creating writer at 1733630495667 (+414 ms)Flushing 1588230740/table: appending metadata at 1733630495683 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733630495683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@365044b2: reopening flushed file at 1733630496090 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ae7228: reopening flushed file at 1733630496096 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67cda41: reopening flushed file at 1733630496099 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@256beb19: reopening flushed file at 1733630496103 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false at 1733630496107 (+4 ms)Writing region close event to WAL at 1733630496114 (+7 ms)Running coprocessor post-close hooks at 1733630496116 (+2 ms)Closed at 1733630496117 (+1 ms) 2024-12-08T04:01:36,117 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:36,178 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,39919,1733630308479; all regions closed. 2024-12-08T04:01:36,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741834_1010 (size=25344) 2024-12-08T04:01:36,190 DEBUG [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/oldWALs 2024-12-08T04:01:36,190 INFO [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C39919%2C1733630308479.meta:.meta(num 1733630312560) 2024-12-08T04:01:36,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741832_1008 (size=17153248) 2024-12-08T04:01:36,195 DEBUG [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/oldWALs 2024-12-08T04:01:36,195 INFO [RS:0;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C39919%2C1733630308479:(num 1733630311545) 2024-12-08T04:01:36,195 DEBUG [RS:0;32df4b4bb439:39919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:36,195 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:36,195 INFO [RS:0;32df4b4bb439:39919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:36,195 INFO [RS:0;32df4b4bb439:39919 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:36,195 INFO [RS:0;32df4b4bb439:39919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:36,195 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:36,196 INFO [RS:0;32df4b4bb439:39919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39919 2024-12-08T04:01:36,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:36,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,39919,1733630308479 2024-12-08T04:01:36,251 INFO [RS:0;32df4b4bb439:39919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:36,259 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32df4b4bb439,39919,1733630308479] 2024-12-08T04:01:36,267 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32df4b4bb439,39919,1733630308479 already deleted, retry=false 2024-12-08T04:01:36,267 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32df4b4bb439,39919,1733630308479 expired; onlineServers=0 2024-12-08T04:01:36,267 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32df4b4bb439,38065,1733630307764' ***** 2024-12-08T04:01:36,267 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T04:01:36,267 INFO [M:0;32df4b4bb439:38065 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:36,267 INFO [M:0;32df4b4bb439:38065 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:36,268 DEBUG [M:0;32df4b4bb439:38065 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T04:01:36,268 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T04:01:36,268 DEBUG [M:0;32df4b4bb439:38065 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T04:01:36,268 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630311263 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630311263,5,FailOnTimeoutGroup] 2024-12-08T04:01:36,268 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630311261 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630311261,5,FailOnTimeoutGroup] 2024-12-08T04:01:36,268 INFO [M:0;32df4b4bb439:38065 {}] hbase.ChoreService(370): Chore service for: master/32df4b4bb439:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:36,268 INFO [M:0;32df4b4bb439:38065 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:36,268 DEBUG [M:0;32df4b4bb439:38065 {}] master.HMaster(1795): Stopping service threads 2024-12-08T04:01:36,268 INFO [M:0;32df4b4bb439:38065 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T04:01:36,268 INFO [M:0;32df4b4bb439:38065 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T04:01:36,268 ERROR [M:0;32df4b4bb439:38065 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-08T04:01:36,269 INFO [M:0;32df4b4bb439:38065 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T04:01:36,269 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T04:01:36,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:36,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:36,275 DEBUG [M:0;32df4b4bb439:38065 {}] zookeeper.ZKUtil(347): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T04:01:36,276 WARN [M:0;32df4b4bb439:38065 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T04:01:36,276 INFO [M:0;32df4b4bb439:38065 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/.lastflushedseqids 2024-12-08T04:01:36,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742445_1621 (size=72) 2024-12-08T04:01:36,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:36,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a0e9140001, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:36,362 INFO [RS:0;32df4b4bb439:39919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:36,362 INFO [RS:0;32df4b4bb439:39919 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,39919,1733630308479; zookeeper connection closed. 2024-12-08T04:01:36,363 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35d5afc2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35d5afc2 2024-12-08T04:01:36,364 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T04:01:36,689 INFO [M:0;32df4b4bb439:38065 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T04:01:36,690 INFO [M:0;32df4b4bb439:38065 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T04:01:36,690 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T04:01:36,690 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:36,690 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:36,690 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T04:01:36,691 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:36,691 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=782.99 KB heapSize=959.89 KB 2024-12-08T04:01:36,713 DEBUG [M:0;32df4b4bb439:38065 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b9ca2121cda4974ad10edc3723a41b4 is 82, key is hbase:meta,,1/info:regioninfo/1733630312635/Put/seqid=0 2024-12-08T04:01:36,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742446_1622 (size=5672) 2024-12-08T04:01:37,117 INFO [M:0;32df4b4bb439:38065 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2187 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b9ca2121cda4974ad10edc3723a41b4 2024-12-08T04:01:37,175 DEBUG [M:0;32df4b4bb439:38065 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/306383d5e15242fb853416c17310711d is 2480, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1733630343298/Put/seqid=0 2024-12-08T04:01:37,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742447_1623 (size=45403) 2024-12-08T04:01:37,580 INFO [M:0;32df4b4bb439:38065 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=782.43 KB at sequenceid=2187 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/306383d5e15242fb853416c17310711d 2024-12-08T04:01:37,591 INFO [M:0;32df4b4bb439:38065 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 306383d5e15242fb853416c17310711d 2024-12-08T04:01:37,604 DEBUG [M:0;32df4b4bb439:38065 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/34021b3de4cb4f358b11a6ad1aa7271e is 69, key is 32df4b4bb439,39919,1733630308479/rs:state/1733630311292/Put/seqid=0 2024-12-08T04:01:37,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073742448_1624 (size=5156) 2024-12-08T04:01:38,008 INFO [M:0;32df4b4bb439:38065 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2187 (bloomFilter=true), to=hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/34021b3de4cb4f358b11a6ad1aa7271e 2024-12-08T04:01:38,013 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b9ca2121cda4974ad10edc3723a41b4 as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b9ca2121cda4974ad10edc3723a41b4 2024-12-08T04:01:38,018 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b9ca2121cda4974ad10edc3723a41b4, entries=8, sequenceid=2187, filesize=5.5 K 2024-12-08T04:01:38,019 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/306383d5e15242fb853416c17310711d as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/306383d5e15242fb853416c17310711d 2024-12-08T04:01:38,023 INFO [M:0;32df4b4bb439:38065 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 306383d5e15242fb853416c17310711d 2024-12-08T04:01:38,023 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/306383d5e15242fb853416c17310711d, entries=170, sequenceid=2187, filesize=44.3 K 2024-12-08T04:01:38,024 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/34021b3de4cb4f358b11a6ad1aa7271e as hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/34021b3de4cb4f358b11a6ad1aa7271e 2024-12-08T04:01:38,029 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37247/user/jenkins/test-data/160d0fb6-7bdd-abe1-0834-3e8294e6f384/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/34021b3de4cb4f358b11a6ad1aa7271e, entries=1, sequenceid=2187, filesize=5.0 K 2024-12-08T04:01:38,030 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(3140): Finished flush of dataSize ~782.99 KB/801780, heapSize ~959.59 KB/982624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1339ms, sequenceid=2187, compaction requested=false 2024-12-08T04:01:38,032 INFO [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:38,032 DEBUG [M:0;32df4b4bb439:38065 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733630496690Disabling compacts and flushes for region at 1733630496690Disabling writes for close at 1733630496690Obtaining lock to block concurrent updates at 1733630496691 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733630496691Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=801780, getHeapSize=982864, getOffHeapSize=0, getCellsCount=2382 at 1733630496693 (+2 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733630496694 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733630496694Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733630496713 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733630496713Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733630497162 (+449 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733630497175 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733630497175Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733630497591 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733630497604 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733630497604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635efd6a: reopening flushed file at 1733630498012 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4985bb94: reopening flushed file at 1733630498018 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a42b4d6: reopening flushed file at 1733630498023 (+5 ms)Finished flush of dataSize ~782.99 KB/801780, heapSize ~959.59 KB/982624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1339ms, sequenceid=2187, compaction requested=false at 1733630498030 (+7 ms)Writing region close event to WAL at 1733630498031 (+1 ms)Closed at 1733630498031 2024-12-08T04:01:38,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37085 is added to blk_1073741830_1006 (size=945134) 2024-12-08T04:01:38,034 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:38,034 INFO [M:0;32df4b4bb439:38065 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T04:01:38,034 INFO [M:0;32df4b4bb439:38065 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38065 2024-12-08T04:01:38,034 INFO [M:0;32df4b4bb439:38065 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:38,097 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T04:01:38,098 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T04:01:38,098 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T04:01:38,099 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T04:01:38,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:38,192 INFO [M:0;32df4b4bb439:38065 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:38,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38065-0x10003a0e9140000, quorum=127.0.0.1:55651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:38,200 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T04:01:38,203 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T04:01:38,204 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T04:01:38,204 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T04:01:38,204 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/hadoop.log.dir/,STOPPED} 2024-12-08T04:01:38,207 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T04:01:38,207 WARN [BP-1961173970-172.17.0.2-1733630304378 heartbeating to localhost/127.0.0.1:37247 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T04:01:38,207 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T04:01:38,207 WARN [BP-1961173970-172.17.0.2-1733630304378 heartbeating to localhost/127.0.0.1:37247 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1961173970-172.17.0.2-1733630304378 (Datanode Uuid 3a01745f-8c30-4451-ba15-ac95fbb8d88b) service to localhost/127.0.0.1:37247 2024-12-08T04:01:38,209 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/data/data1/current/BP-1961173970-172.17.0.2-1733630304378 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T04:01:38,209 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/cluster_371257fb-746a-c60a-6859-9ca3fce21dff/data/data2/current/BP-1961173970-172.17.0.2-1733630304378 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T04:01:38,210 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T04:01:38,216 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T04:01:38,217 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T04:01:38,217 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T04:01:38,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T04:01:38,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/012ee34c-fc12-ad33-d5e1-b7a479cf00c9/hadoop.log.dir/,STOPPED} 2024-12-08T04:01:38,231 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T04:01:38,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down